{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic 回归——Otto商品分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，分别调用缺省参数LogisticRegression、LogisticRegression + GridSearchCV以及LogisticRegressionCV进行参数调优。实际应用中LogisticRegression + GridSearchCV或LogisticRegressionCV任选一个即可。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n",
    "\n",
    "\n",
    "第一名：https://www.kaggle.com/c/otto-group-product-classification-challenge/discussion/14335\n",
    "第二名：http://blog.kaggle.com/2015/06/09/otto-product-classification-winners-interview-2nd-place-alexander-guschin/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#竞赛的评价指标为logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1  feat_2  feat_3  feat_4  feat_5  feat_6  feat_7  feat_8  feat_9  \\\n",
       "0   1       1       0       0       0       0       0       0       0       0   \n",
       "1   2       0       0       0       0       0       0       0       1       0   \n",
       "2   3       0       0       0       0       0       0       0       1       0   \n",
       "3   4       1       0       0       1       6       1       5       0       0   \n",
       "4   5       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "    ...     feat_85  feat_86  feat_87  feat_88  feat_89  feat_90  feat_91  \\\n",
       "0   ...           1        0        0        0        0        0        0   \n",
       "1   ...           0        0        0        0        0        0        0   \n",
       "2   ...           0        0        0        0        0        0        0   \n",
       "3   ...           0        1        2        0        0        0        0   \n",
       "4   ...           1        0        0        0        0        1        0   \n",
       "\n",
       "   feat_92  feat_93   target  \n",
       "0        0        0  Class_1  \n",
       "1        0        0  Class_1  \n",
       "2        0        0  Class_1  \n",
       "3        0        0  Class_1  \n",
       "4        0        0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = '../data/'\n",
    "train = pd.read_csv(dpath +\"Otto_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 61878 entries, 0 to 61877\n",
      "Data columns (total 95 columns):\n",
      "id         61878 non-null int64\n",
      "feat_1     61878 non-null int64\n",
      "feat_2     61878 non-null int64\n",
      "feat_3     61878 non-null int64\n",
      "feat_4     61878 non-null int64\n",
      "feat_5     61878 non-null int64\n",
      "feat_6     61878 non-null int64\n",
      "feat_7     61878 non-null int64\n",
      "feat_8     61878 non-null int64\n",
      "feat_9     61878 non-null int64\n",
      "feat_10    61878 non-null int64\n",
      "feat_11    61878 non-null int64\n",
      "feat_12    61878 non-null int64\n",
      "feat_13    61878 non-null int64\n",
      "feat_14    61878 non-null int64\n",
      "feat_15    61878 non-null int64\n",
      "feat_16    61878 non-null int64\n",
      "feat_17    61878 non-null int64\n",
      "feat_18    61878 non-null int64\n",
      "feat_19    61878 non-null int64\n",
      "feat_20    61878 non-null int64\n",
      "feat_21    61878 non-null int64\n",
      "feat_22    61878 non-null int64\n",
      "feat_23    61878 non-null int64\n",
      "feat_24    61878 non-null int64\n",
      "feat_25    61878 non-null int64\n",
      "feat_26    61878 non-null int64\n",
      "feat_27    61878 non-null int64\n",
      "feat_28    61878 non-null int64\n",
      "feat_29    61878 non-null int64\n",
      "feat_30    61878 non-null int64\n",
      "feat_31    61878 non-null int64\n",
      "feat_32    61878 non-null int64\n",
      "feat_33    61878 non-null int64\n",
      "feat_34    61878 non-null int64\n",
      "feat_35    61878 non-null int64\n",
      "feat_36    61878 non-null int64\n",
      "feat_37    61878 non-null int64\n",
      "feat_38    61878 non-null int64\n",
      "feat_39    61878 non-null int64\n",
      "feat_40    61878 non-null int64\n",
      "feat_41    61878 non-null int64\n",
      "feat_42    61878 non-null int64\n",
      "feat_43    61878 non-null int64\n",
      "feat_44    61878 non-null int64\n",
      "feat_45    61878 non-null int64\n",
      "feat_46    61878 non-null int64\n",
      "feat_47    61878 non-null int64\n",
      "feat_48    61878 non-null int64\n",
      "feat_49    61878 non-null int64\n",
      "feat_50    61878 non-null int64\n",
      "feat_51    61878 non-null int64\n",
      "feat_52    61878 non-null int64\n",
      "feat_53    61878 non-null int64\n",
      "feat_54    61878 non-null int64\n",
      "feat_55    61878 non-null int64\n",
      "feat_56    61878 non-null int64\n",
      "feat_57    61878 non-null int64\n",
      "feat_58    61878 non-null int64\n",
      "feat_59    61878 non-null int64\n",
      "feat_60    61878 non-null int64\n",
      "feat_61    61878 non-null int64\n",
      "feat_62    61878 non-null int64\n",
      "feat_63    61878 non-null int64\n",
      "feat_64    61878 non-null int64\n",
      "feat_65    61878 non-null int64\n",
      "feat_66    61878 non-null int64\n",
      "feat_67    61878 non-null int64\n",
      "feat_68    61878 non-null int64\n",
      "feat_69    61878 non-null int64\n",
      "feat_70    61878 non-null int64\n",
      "feat_71    61878 non-null int64\n",
      "feat_72    61878 non-null int64\n",
      "feat_73    61878 non-null int64\n",
      "feat_74    61878 non-null int64\n",
      "feat_75    61878 non-null int64\n",
      "feat_76    61878 non-null int64\n",
      "feat_77    61878 non-null int64\n",
      "feat_78    61878 non-null int64\n",
      "feat_79    61878 non-null int64\n",
      "feat_80    61878 non-null int64\n",
      "feat_81    61878 non-null int64\n",
      "feat_82    61878 non-null int64\n",
      "feat_83    61878 non-null int64\n",
      "feat_84    61878 non-null int64\n",
      "feat_85    61878 non-null int64\n",
      "feat_86    61878 non-null int64\n",
      "feat_87    61878 non-null int64\n",
      "feat_88    61878 non-null int64\n",
      "feat_89    61878 non-null int64\n",
      "feat_90    61878 non-null int64\n",
      "feat_91    61878 non-null int64\n",
      "feat_92    61878 non-null int64\n",
      "feat_93    61878 non-null int64\n",
      "target     61878 non-null object\n",
      "dtypes: int64(94), object(1)\n",
      "memory usage: 44.8+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_84</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.00000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61878.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>30939.500000</td>\n",
       "      <td>0.38668</td>\n",
       "      <td>0.263066</td>\n",
       "      <td>0.901467</td>\n",
       "      <td>0.779081</td>\n",
       "      <td>0.071043</td>\n",
       "      <td>0.025696</td>\n",
       "      <td>0.193704</td>\n",
       "      <td>0.662433</td>\n",
       "      <td>1.011296</td>\n",
       "      <td>...</td>\n",
       "      <td>0.070752</td>\n",
       "      <td>0.532306</td>\n",
       "      <td>1.128576</td>\n",
       "      <td>0.393549</td>\n",
       "      <td>0.874915</td>\n",
       "      <td>0.457772</td>\n",
       "      <td>0.812421</td>\n",
       "      <td>0.264941</td>\n",
       "      <td>0.380119</td>\n",
       "      <td>0.126135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>17862.784315</td>\n",
       "      <td>1.52533</td>\n",
       "      <td>1.252073</td>\n",
       "      <td>2.934818</td>\n",
       "      <td>2.788005</td>\n",
       "      <td>0.438902</td>\n",
       "      <td>0.215333</td>\n",
       "      <td>1.030102</td>\n",
       "      <td>2.255770</td>\n",
       "      <td>3.474822</td>\n",
       "      <td>...</td>\n",
       "      <td>1.151460</td>\n",
       "      <td>1.900438</td>\n",
       "      <td>2.681554</td>\n",
       "      <td>1.575455</td>\n",
       "      <td>2.115466</td>\n",
       "      <td>1.527385</td>\n",
       "      <td>4.597804</td>\n",
       "      <td>2.045646</td>\n",
       "      <td>0.982385</td>\n",
       "      <td>1.201720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>15470.250000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>30939.500000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>46408.750000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>61878.000000</td>\n",
       "      <td>61.00000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>43.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>65.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>61.000000</td>\n",
       "      <td>130.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>87.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 94 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id       feat_1        feat_2        feat_3        feat_4  \\\n",
       "count  61878.000000  61878.00000  61878.000000  61878.000000  61878.000000   \n",
       "mean   30939.500000      0.38668      0.263066      0.901467      0.779081   \n",
       "std    17862.784315      1.52533      1.252073      2.934818      2.788005   \n",
       "min        1.000000      0.00000      0.000000      0.000000      0.000000   \n",
       "25%    15470.250000      0.00000      0.000000      0.000000      0.000000   \n",
       "50%    30939.500000      0.00000      0.000000      0.000000      0.000000   \n",
       "75%    46408.750000      0.00000      0.000000      0.000000      0.000000   \n",
       "max    61878.000000     61.00000     51.000000     64.000000     70.000000   \n",
       "\n",
       "             feat_5        feat_6        feat_7        feat_8        feat_9  \\\n",
       "count  61878.000000  61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       0.071043      0.025696      0.193704      0.662433      1.011296   \n",
       "std        0.438902      0.215333      1.030102      2.255770      3.474822   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      1.000000      0.000000   \n",
       "max       19.000000     10.000000     38.000000     76.000000     43.000000   \n",
       "\n",
       "           ...            feat_84       feat_85       feat_86       feat_87  \\\n",
       "count      ...       61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       ...           0.070752      0.532306      1.128576      0.393549   \n",
       "std        ...           1.151460      1.900438      2.681554      1.575455   \n",
       "min        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "25%        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "50%        ...           0.000000      0.000000      0.000000      0.000000   \n",
       "75%        ...           0.000000      0.000000      1.000000      0.000000   \n",
       "max        ...          76.000000     55.000000     65.000000     67.000000   \n",
       "\n",
       "            feat_88       feat_89       feat_90       feat_91       feat_92  \\\n",
       "count  61878.000000  61878.000000  61878.000000  61878.000000  61878.000000   \n",
       "mean       0.874915      0.457772      0.812421      0.264941      0.380119   \n",
       "std        2.115466      1.527385      4.597804      2.045646      0.982385   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        1.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max       30.000000     61.000000    130.000000     52.000000     19.000000   \n",
       "\n",
       "            feat_93  \n",
       "count  61878.000000  \n",
       "mean       0.126135  \n",
       "std        1.201720  \n",
       "min        0.000000  \n",
       "25%        0.000000  \n",
       "50%        0.000000  \n",
       "75%        0.000000  \n",
       "max       87.000000  \n",
       "\n",
       "[8 rows x 94 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEFCAYAAAD0cwBnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHoFJREFUeJzt3XuYXFWZ7/FvJ50LgU5sJRE8Rw2g/kAGhAcFlASCBjAM\nxyiicrgrEy4HDXg5IBIUPCDCKArOCDMR5RK8jEFEg+EiMZBEIEeMkAzyYoIOqAFbzKUxJDFJzx9r\nlymapHrvTlft6vTv8zw81F61qtbbVZV6a+219lotXV1dmJmZ5TWo7ADMzKx/ceIwM7NCnDjMzKwQ\nJw4zMyvEicPMzAppLTuAeuvo6PS0MTOzgkaPbmvZ2n3ucZiZWSFOHGZmVogTh5mZFeLEYWZmhThx\nmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhdb1yXNJBwJURMUHSGGA60A4MBk6JiGWSpgBnAhuA\nyyJilqQdgBnAGKATODUiOiQdDFyT1b0nIi6tZ/xmZvZydUscks4HTgb+mhVdBdwaEf8h6XBgT0l/\nBaYCbwWGA/Ml3QucDSyOiEskHQ9MA84FrgfeDzwF3Clp/4hYVK+/oVGWzzyxlHZ3Pe7WUto1s/6t\nnj2OZcCxwC3Z8SHAY5J+CvyOlAjeBSyIiHXAOklLgX2BcaREAzAbuFjSSGBYRCwDkHQ3MBGomTja\n20fQ2jq4L/+uPre8pHZHj24rqWUz68/qljgi4jZJY6uKxgIrImKipM8CFwBPAquq6nQCo4CRVeXV\nZau71d29pzhWrFjTy79g+9fR0Vl2CGbWpGr9sGzk4PjzwI+y2z8mnZ5aDVRH1was7Fa+pbLqcjMz\na6BGJo75wNHZ7UOB/wQWAuMlDZc0CtgLWAIsqKo7CZgXEauB9ZL2kNQCHAXMa2D8ZmZGYxPHJ4FT\nJP0ceDfwhYh4FriWlADmABdFxFrgOmBvSfOBM4DK7KmzgFtJCWdRRDzcwPjNzAxo6eravvc56g8b\nOXlWlZk1G2/kZGZmfcaJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4z\nMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrpLWeTy7pIODK\niJhQVXYC8LGIeHt2PAU4E9gAXBYRsyTtAMwAxgCdwKkR0SHpYOCarO49EXEpZmbWUHVLHJLOB04G\n/lpVtj9wOtCSHe8CTAXeCgwH5ku6FzgbWBwRl0g6HpgGnAtcD7wfeAq4U9L+EbGoXn+DWX91+v33\nl9LuDYcdVkq71lj1PFW1DDi2ciDpVcAXgPOq6hwILIiIdRGxClgK7AuMA+7K6swGJkoaCQyLiGUR\n0QXcDUysY/xmZrYFdetxRMRtksYCSBoM3AB8AnixqtpIYFXVcScwqlt5ddnqbnV37ymO9vYRtLYO\n7t0f0SDLS2p39Oi2klq27ZU/UwNDXcc4qhwAvBG4jnRK6s2SvgrMAao/aW3ASlKCaKtRVl1e04oV\na7Y19u1WR0dn2SHYdsafqe1HrR8BDUkcEbEQ2Bsg64V8NyLOy8Y4Lpc0HBgG7AUsARYARwMLgUnA\nvIhYLWm9pD1IYxxHAR4cNzNrsFKn40bEs8C1wDxS7+OiiFhL6pnsLWk+cAabE8RZwK2khLIoIh5u\nfNRmZgNbS1dXV9kx1FVHR2fT/4HLZ55YSru7HndrKe1a/XlWlW2r0aPbWrZ2ny8ANDOzQpw4zMys\nECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4zMyvEicPMzApx4jAzs0KcOMzMrJBciUPS0Oz/b5D0\nj5KccMzMBqgeE4CkzwLfkPQ64AHg48C/1TswMzNrTnl6Du8BpgAnADMiYiKwf12jMjOzppUncQyO\niHXAMcBPstNUO9Y3LDMza1Z5Esd9kpYAQ0mnqu4HflTXqMzMrGn1mDgi4lOk3fgOjohNwMci4oK6\nR2ZmZk0pz+B4O3Ax8FNJrwKmZmVmZjYA5TlVNR34/8CrgE5gOTCjnkGZmVnzas1RZ7eI+HdJZ0fE\neuAiSY/meXJJBwFXRsQESfsBXwM2AuuAUyLiOUlTgDOBDcBlETFL0g6k5DSGlKxOjYgOSQcD12R1\n74mIS7fQrJmZ1VGeHscGSaOALgBJbwQ29fQgSecD3wCGZ0XXkMZHJgA/AC6QtAswFTgEOAq4QtIw\n4GxgcUSMB24GpmXPcT1pWvA44CBJnhZsZtZgeRLH54C5wOsl/RCYz+Yv8lqWAcdWHR8fEb/KbrcC\na4EDgQURsS4iVgFLgX1JieGurO5sYKKkkcCwiFgWEV3A3cDEHHGYmVkf6vFUVUTcJekXwEHAYODM\niHgux+NukzS26ng5gKR3AB8FDiX1MlZVPawTGAWMrCqvLlvdre7uPcXR3j6C1tbBPVUr1fKS2h09\nuq2klm175c/UwNBj4pB0OGns4RBJAh6UdFJE/LxoY5I+BFwE/GM2ZrEaqP6ktQErSQmirUZZdXlN\nK1asKRrmgNHR0Vl2CLad8Wdq+1HrR0CeU1VfJg1eExFBuqbjmqJBSDqJ1NOYEBFPZcULgfGShmfj\nKHsBS4AFWTsAk4B5EbEaWC9pD0ktpN7KvKJxmJnZtsmTOIZHxJLKQUQ8AQwp0oikwcC1pF7CDyTN\nlXRpRDyblc8D5gAXRcRa4Dpgb0nzgTOAyuyps4BbSQlnUUQ8XCQOMzPbdnmm4z4h6Urgluz4eODJ\nPE8eEb8DDs4OX7mVOtNJ14pUl60BPrCFug9VPZ+ZmZUgT4/jdGAn4DukqbE7kVbLNTOzASjPrKoV\nwDkNiMXMzPqBPLOqTgO+BFTWp2oBuiKiuee4mplZXeQZ4/gsaSbUkh5rmpnZdi/PGMcfnDTMzKwi\nT4/jEUkzgXtIy4QAEBE31y0qMzNrWnkSxyjS8h5vryrrIs2wMjOzASbPrKoPQ9rQKZthZWZmA1ie\nWVVvAb4HjMj2w3gA+GBE/LLewZmZWfPJMzj+NeB9wPMR8UfSXhnX1zUqMzNrWnkSx4iI+HXlICLu\nBYbVLyQzM2tmeRLHX7LTVZUdAE8E/lLXqMzMrGnlmVV1NnATabXalcBvgBPrGpWZmTWtPInjiIgY\nJ2lHYHC2L4aZmQ1QeRLHR4HrI+Kv9Q7GzMyaX57E8YykOcDDwIuVwoj4fN2iMjOzppUncTxUdbul\nXoGYmVn/kCdxjK1cPW5mZpYncfyDpJ0i4oWiTy7pIODKiJgg6Q3AjaRpvUuAcyJik6QpwJnABuCy\niJglaQdgBjCGtE7WqRHRkV25fk1W956IuPTlrZqZWT3luY5jE/C0pAclzan819ODJJ0PfAMYnhVd\nDUyLiPGkU16TJe0CTAUOAY4CrpA0jDQFeHFW92ZgWvYc1wMnAOOAgyTtn/cPNTOzvpGnx3F+L597\nGXAscEt2fABwf3Z7NnAksBFYEBHrgHWSlgL7khLDVVV1L5Y0EhgWEcsAJN0NTAQW1QqivX0Era3N\nvVnh8pLaHT26raSWbXvlz9TAkCdxdPXmiSPiNkljq4paIqLyXJ2k5dpHAquq6mypvLpsdbe6u/cU\nx4oVa3oT/oDQ0dFZdgi2nfFnavtR60dAnsRRPY4whNQjmEdaJbeITVW324CVpETQ1kN5T3XNzKyB\n8uzHcXj1saTdgK/0oq1FkiZExFxgEvAzYCFwuaThpIUT9yINnC8Ajs7unwTMi4jVktZL2gN4ijQm\n4sFxM7MGyzM4/hIR8Vtgz1609UngUkkPAkOBmRHxLHAtqQczB7goItYC15HWxpoPnMHmBHEWcCsp\noSyKiId7EYeZmW2Dlq6u2kMYkr7F5nGOFlKv4PcRcVydY+sTHR2dvRqjaaTlM8tZM3LX424tpV2r\nv9Pvv7/nSnVww2GHldKu9b3Ro9u2esF3njGOuVW3u4DvAz/dxpjMzKyfynOq6gfAThFxE3AfcDD5\nEo6ZmW2H8iSOW4Fds9ud2WNu2Xp1MzPbnuXpObw+It4DkO3FMU3Sr+oblpmZNas8PY4uSftUDiTt\nCfytfiGZmVkzy9Pj+BRwr6Tfk2ZV7QycVNeozMysaeW5APCnkl4H7EPqaSzzboBmZgNXj6eqJH0Q\neCQiHgH+CjwhaXLdIzMzs6aUZ4xjGmkVWrKVaQ/AS32YmQ1YeRLH0Ih4rnIQEX/CW8iamQ1YeQbH\n50v6Dul6DoAPAg/WLyQzM2tmeRLHOcDHSNu7rictSPj1egZlZmbNK8+sqnWS7gLWAYOBuRGxvu6R\nmZlZU8ozq+pk4A5gLPB64AeSPlLnuMzMrEnlOVX1SeDAiHgeQNLlpBVzv1nHuMzM6u7J2WtLafdN\nk4aX0m5fyTOranAlaQBExJ956TawZmY2gOTpcTwq6avADdnx6cCj9QvJzMyaWZ7EMQW4hHRqahBp\nT47/05vGJA0BbiKNl2zMnnsDcCNpk6glwDkRsUnSFNJMrg3AZRExS9IOwAxgDGmJ91MjoqM3sZiZ\nWe/kmVX1InBBH7V3NNAaEe+QdARwOTAEmBYRcyVdD0zO9iWfCrwVGE66luRe4GxgcURcIul40lXt\n5/ZRbGZmlkOeMY6+9CTQKmkQMJK0aOIBQGWD5Nmk5U0OBBZExLqIWAUsBfYFxgF3datrZmYNtNUe\nh6Qd67AK7guk01RPkJZnPwY4NCK6svs7gVGkpLKq6nFbKq+U1dTePoLW1sF9EXvdLC+p3dGj27Z6\n3y03HtXASDY7+bS7S2nX+katz1QzepJyZlX1t9epu1qnquYCb5P09Yjo1ZjGFnwcuDsiLpT0WmAO\nMLTq/jZgJbA6u12rvFJW04oVa/og7O1TR0dn2SG8TDPGZPn5/cunP7xOtZJbrcSxk6QZwLslvWzS\ncUT05iLAFWzePfAvpPGNRZImRMRcYBLwM2AhcHnW7jBgL9LA+QLSOMnCrO68XsRgZmbboFbiOBI4\nHBjP5jGIbfUV4JuS5pF6Gp8BfgFMlzQU+DUwMyI2SrqWlBgGARdFxFpJ1wE3SZpPWjfrhD6Ky8zM\nctpq4oiIZ4CbJT0KPA4oq78kIjb0prGIeIG0um53h22h7nRgereyNcAHetO2mZn1jTyzqoYAvyFd\nf/Et4GlJB9U1KjMza1p5LgC8BvhQRDwMIOlg4GukKbNmZjbA5Olx7FRJGgAR8RDpojwzMxuA8iSO\nv0iaXDmQ9F7g+Rr1zcxsO5bnVNUZwAxJN5D2Gl8GnFTXqMzMrGnlWavqN8BBknYEBkVE81+5YmZm\ndZOnxwFAHZYfMTOzfqjRixyamVk/l2fP8bMaEYiZmfUPeXocH617FGZm1m/kGeN4RtIc4GHgxUph\nRHy+blGZmVnTypM4Hqq63VKvQMzMrH/IMx330mwq7h6kpc138AwrM7OBK8/g+DuBR4E7gFcDv5N0\nZL0DMzOz5pRncPwK0l7fKyNiOWkJ9H+ua1RmZta08iSOQRHxbOUgIh6vYzxmZtbk8gyO/17SMUCX\npFcA5wBP1zcsMzNrVnl6HGcCJwKvBZ4C9iMtfGhmZgNQnllVfwL+t6SRwN8i4sWeHlOLpAuB95D2\nHP86aT/zG4Eu0qytcyJik6QppKS1AbgsImZJ2gGYAYwBOoFTI6JjW+IxM7Ni8syq2kfSL0m9jWck\nzZe0R28akzQBeAdwCGmQ/bXA1cC0iBhPuk5ksqRdgKlZvaOAKyQNA84GFmd1bwam9SYOMzPrvTyn\nqq4HLoqInSNiZ+DLwDd72d5RwGLgduDHwCzgAFKvA2A2MJG0Le2CiFgXEauApcC+pNldd3Wra2Zm\nDZRncHyHiJhdOYiI2yV9tpft7Qy8HjgG2A34EWnWVld2fycwChgJrKp63JbKK2U1tbePoLV1cC/D\nbYzlJbU7enRbSS1vXTPGZPn1t/fvSdaW0m5/e52622rikPS67Oajkj4N3EAabzgRmNfL9p4HnoiI\n9UBIWks6XVXRBqwEVme3a5VXympasWJNL0Pd/nV0NN+eXM0Yk+Xn9y+f/vA61UputXoc95MGrFuA\nCaSB6oou0hhEUfOBcyVdDewK7AjcJ2lCRMwFJgE/AxYCl0saDgwD9iINnC8Ajs7un0TvE5iZmfXS\nVhNHROzW141lM6MOJX3xDyJdE/JbYLqkocCvgZkRsVHStaTEMIg0xrJW0nXATZLmA+uBE/o6RjMz\nq63HMQ5JIl230V5dHhEf6U2DEXH+FooP20K96cD0bmVrgA/0pl0zM+sbeQbHbwe+CzxW51jMzKwf\nyJM4VnrTJjMzq8iTOG6UdDlwH2lWFQAR8UDdojIzs6aVJ3FMAN5GuuK7ogt4Zz0CMjOz5pYncbw1\nIt5Y90jMzKxfyLPkyGJJ+9Y9EjMz6xfy9Dh2BxZJWk66dqIF6IqI3esamZmZNaU8ieO9dY/CzMz6\njTyJ42UX52Vu7stAzMws2TTjtw1vc9BJ+RcLyZM4Dq+6PQQYDzyAE4eZ2YCUZwfAD1cfS3ol8L26\nRWRmZk0tz6yq7l4AxvZxHGZm1k/kWeTwZ6QL/iDNqNoduLOeQZmZWfPKM8ZxSdXtLuDPEfF4fcIx\nM7Nml2cHwJcN70t6XUQ8XbeozMysaeXdAbCiC3gNaXZVc2/kbWZmdZF7B0BJOwFfBo4CptQ5LjMz\na1K5ZlVJehebN3LaJyLurV9IZmbWzGoOjkvaEbiarJfRVwlD0hjgEeAI0h4fN5JOgy0BzomITZKm\nAGdm91+W7Ve+AzADGAN0AqdGREdfxGRmZvlstceR9TIWZ4f/0IdJYwjwb8CLWdHVwLSIGE8aT5ks\naRdgKnAIKWldIWkYcDawOKt7MzCtL2IyM7P8avU47gX+BhwJPCapUr6tq+N+CbgeuDA7PoA0EA8w\nO2tvI7AgItYB6yQtBfYFxgFXVdW9uKfG2ttH0Nra3OP4y0tqd/TotpJa3rpmjMny62/v35OsLaXd\nnl6n5xoUR7Ui712txJF/xaucJJ0GdETE3ZIqiaMlIioXGHYCo4CRwKqqh26pvFJW04oVa/og8u1T\nR0dn2SG8TDPGZPn5/cunGV+n7jHVSiS1ZlX9V9+F9HcfAbokTQT2I51uGlN1fxuwElid3a5VXikz\nM7MG6s1aVb0WEYdGxGERMQH4FXAKMFvShKzKJGAesBAYL2m4pFHAXqSB8wXA0d3qmplZAzU0cWzF\nJ4FLJT0IDAVmRsSzwLWkxDAHuCgi1gLXAXtLmg+cAVxaUsxmZgNWnrWq6iLrdVS8bLOoiJgOTO9W\ntgb4QH0jMzOzWpqhx2FmZv2IE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZ\nIU4cZmZWSGlXjpvZwPL/5pezJunF415RSrvbM/c4zMysEPc4rF85f37jlyq7atz3a95/2rxbGhTJ\nS904/uRS2jVzj8PMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCmnorCpJQ4BvAmOBYcBlwOPAjUAX\naV/xcyJik6QpwJnABuCyiJglaQdgBjAG6AROjYiORv4NZmYDXaN7HCcBz0fEeODdwL8AVwPTsrIW\nYLKkXYCpwCHAUcAVkoYBZwOLs7o3A9MaHL+Z2YDX6MTxfeDi7HYLqTdxAHB/VjYbmAgcCCyIiHUR\nsQpYCuwLjAPu6lbXzMwaqKGnqiLiBQBJbcBMUo/hSxHRlVXpBEYBI4FVVQ/dUnmlrKb29hG0tg7u\nk/jrZXlJ7Y4e3VZSy1vnmPJrxrhqx1TOkiO1YnqStQ2MZLOe3rvnGhRHtSKfp4ZfOS7ptcDtwNcj\n4tuSrqq6u4306Vqd3a5VXimracWKNS8tmHlHb0PfNsdNLqfdGjo6OssO4WUcU37NGJdjyqc/xFQr\nkTT0VJWkVwP3ABdExDez4kWSJmS3JwHzgIXAeEnDJY0C9iINnC8Aju5W18zMGqjRPY7PAO3AxZIq\nYx3nAtdKGgr8GpgZERslXUtKDIOAiyJiraTrgJskzQfWAyc0OH4zswGv0WMc55ISRXeHbaHudGB6\nt7I1QONXuTMzs7/zBYBmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZ\nIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZ\nFdLoPce3maRBwNeBtwDrgH+KiKXlRmVmNnD0xx7He4HhEfF24NPAl0uOx8xsQOmPiWMccBdARDwE\nvLXccMzMBpaWrq6usmMoRNI3gNsiYnZ2/DSwe0RsKDcyM7OBoT/2OFYDbVXHg5w0zMwapz8mjgXA\n0QCSDgYWlxuOmdnA0u9mVQG3A0dI+jnQAny45HjMzAaUfjfGYWZm5eqPp6rMzKxEThxmZlaIE4eZ\nmRXSHwfHe0XS3sBVwAhgJ+AnwFzgzIg4vs5tvw/4QEScUHZMkkYBM4CRwFDgExHxYBPEtSPwbaAd\nWA+cGhF/KDOmqrb3BB4GXh0Ra8uMSVIL8HvgN1nRgxFxYckxDQauJl2MOwy4JCJmlRzTp4F3Z4ev\nAHaJiF261Snr3993s/bWASdFxLMlx/RKNn8nPA9MiYg/1XrMgOhxSHoF6c06LyIOBw4G9gHUgLav\nAa6g22tdYkyfAO6LiMOA04B/bZK4pgCPRMShpA/x+U0QE5JGkpa1WdetvKyY9gB+GRETsv+qk0ZZ\nMZ0MDImIQ4DJwBvKjikivlh5jUiJ9pTq+0t8rU4DFkfEeOB7wP9tgpg+A8yPiHHA14Av9PSAgdLj\nmAzMiYjfAETERkmnAO8AJgBI+ihwLLAj8GfgfcBY4FvABtIX/wnAWtIbPggYDpwVEb+q0fbPgR8C\nZzZJTF9h85dga/bY0uOKiK9mv1wBXgesLDum7Nf9v5P+Yd3RDK8TcADwPyT9DHgR+HhERMkxHQUs\nkXQnaYr8x5rgdSJ77mOBFRFxT7e7yoprMbBndnsk8LcmiOnNwEXZ7QXAv2yl3t8NiB4H8BrgqeqC\niHiBdEqksuLuq4CJEXEQ6Qv1bcARwEJgIvA5YBRwIKk7Nwk4h/QGblVEfA/Y0pznUmKKiJUR8aKk\nXUi/7C/sVqXM12qjpDmkL57bmyCmzwF3RsSjW7ivrJiWA1dkv0i/QHoPy45pZ1Iv4xjgStKXWNkx\nVVwIXLqF8rLieh44UtLjpN7GDU0Q06+A92S330M6TVbTQEkc/wW8trpA0m7AoQARsYn05nxH0g3A\n/wSGkN7UlaRFFT9KyuizSVn5DuDzwKb+FpOkfYD7gM9ExP3NElf2/O8ExgO3NUFMJwGnS5oL7AJU\n/2otK6ZfZPWIiPnAa7KeUZkxPQ/Mioiu7PP0piZ4nZD0ZmDlVrZdKCuuzwFXRcSbgSNpjs/5FcBY\nSQ+Qei/P1KgLDJzEMQt4t6Q9ACQNIQ3m/Tk73hd4b0R8iPRrdxCpyz0ZmBcR7wK+D1xA6jIuj4gj\ngcvIcT6wmWLK/jF9HzihslBkk8R1oaSTs8MXgI1lxxQRb6g6T/4s6R96qTGRvnjOy9p4C/BMRFR6\ntGXFNJ/NywC9BXi6CV4nSL/At/QZLzOuFcCq7PafSKeryo7pUGB6Nr64lJRwahowV45LOgD4Z9KL\n3Qb8GLifNPbwEdKbNiyrvo6UxR8CbiJl+cHAx0m/Cr5LyvStwOe3cP60e9sTSOcYj+9W3vCYJN1B\n2gTrd1nRqoiY3ARxvTp7/PDs8Z+OiAVV95f2/mXt/w7YM146q6qM16mddHpqJ9Ivy3Mi4omSYxoG\nXEc6V94CnB0RvywzpqzdfwXujYgfbuX+Ml6r1wDfIL1/Q4DPRsS9Jcf0BuDm7PAPwOkRsXpLdSsG\nTOIwM7O+MVBmVdWVpB8Ar+xW/LJf8o3UjDFBc8blmPJxTPk1Y1x9GZN7HGZmVshAGRw3M7M+4sRh\nZmaFOHGYmVkhThxmfUDSKElbnPbZh218S9Lr69mGWR5OHGZ9ox3Yr85tHE66TsKsVJ5VZdYHJP2I\ntIz3ncDjwLtIUx//DBwbEc9K6gAeIS1f8jbSUhDHZXWWAz+KiBuzhe3OI/2we4S01tB5Wf2lwPiI\neL6Bf57ZS7jHYdY3pgJ/JC1ctyfwjoh4E+mL/sSszs7AFyNiP1KSGQfsTVquY3/4+34MU7LH70da\nluJTEfHF7PmPdtKwsvkCQLM+FBFLJX0S+CdJAt4OLKuq8nD2/yOA/4iI9cD6qvGRw4E3Ag+lhzMU\n+CVmTcSJw6wPZWsNfYe0ON1M0mKNfx+XiIgXs5sb2XKPfzApoUzNnm8n/O/UmoxPVZn1jQ2kL/jD\ngLkRcT1prONIUjLo7l7g/ZKGKu0yeAxp35a5wPskjcmWS7+ObDXcqjbMSuXEYdY3niMtJ/6/gLdI\negyYAzwG7Na9ckT8BHgAWEQaUP8j8GK2adSl2WP/k/Rv9IvZw2YBP8n2aDArjWdVmZVA0tuBN0XE\nTdm+Cw8CH4mIx0oOzaxHThxmJZD0SuDbwK6kXsVNEfGlcqMyy8eJw8zMCvEYh5mZFeLEYWZmhThx\nmJlZIU4cZmZWiBOHmZkV8t8KhP35E+3DFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x106b86710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target 分布，看看各类样本分布是否均衡\n",
    "sns.d(train.target);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡。交叉验证对分类任务缺省的是采用StratifiedKFold，在每折采样时根据各类样本按比例采样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "y_train = train['target']   #形式为Class_x\n",
    "y_train = y_train.map(lambda s: s[6:])\n",
    "y_train = y_train.map(lambda s: int(s)-1)\n",
    "\n",
    "train = train.drop([\"id\", \"target\"], axis=1)\n",
    "X_train = np.array(train)\n",
    "\n",
    "#如果计算资源有限，也可只取少量样本，如取前1000个样本\n",
    "#（分类中其实还需要确保取出来的这部分样本各类样本的比例和总体一致）\n",
    "#n_trains = 1000\n",
    "#y_train = train.label.values[:n_trains]\n",
    "\n",
    "#或者考虑用train_test_split而不是交叉验证来验证模型性能"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "#X_test = ss_X.transform(X_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [ 0.67686698  0.68182075  0.66838086  0.6662869   0.67439387]\n",
      "cv logloss is: 0.673549871424\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print 'logloss of each fold is: ',-loss\n",
    "print'cv logloss is:', -loss.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "目标函数为：J = sum(logloss(f(xi), yi)) + C* penalty \n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "设置候选参数集合\n",
    "调用GridSearchCV\n",
    "调用fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  1.93511062,   5.99620657,   5.26897659,  10.01246262,\n",
       "         15.74666104,  15.71642375,  27.90830598,  20.76507196,\n",
       "         37.48180237,  22.21065221,  36.81701975,  22.43034906,\n",
       "         37.2483572 ,  22.94229341]),\n",
       " 'mean_score_time': array([ 0.01501718,  0.01509032,  0.01391201,  0.01647058,  0.01643734,\n",
       "         0.01428461,  0.01579723,  0.01449585,  0.015205  ,  0.0126554 ,\n",
       "         0.01449461,  0.01259732,  0.01442018,  0.0124156 ]),\n",
       " 'mean_test_score': array([-1.177056  , -1.02651724, -0.77504707, -0.76020957, -0.68194978,\n",
       "        -0.68690556, -0.67238102, -0.67355004, -0.67187999, -0.67184425,\n",
       "        -0.67190296, -0.6718333 , -0.67190395, -0.67187363]),\n",
       " 'mean_train_score': array([-1.17597192, -1.02389825, -0.7709288 , -0.75433198, -0.6725468 ,\n",
       "        -0.67728075, -0.66042996, -0.66191617, -0.65922489, -0.65940444,\n",
       "        -0.65913019, -0.65909357, -0.65911419, -0.65906463]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 13, 12, 11,  9, 10,  7,  8,  4,  2,  5,  1,  6,  3], dtype=int32),\n",
       " 'split0_test_score': array([-1.18016728, -1.02714364, -0.77838107, -0.76291001, -0.68500844,\n",
       "        -0.69036094, -0.67590255, -0.67686698, -0.67555642, -0.67545109,\n",
       "        -0.67555681, -0.67548819, -0.67554653, -0.67551303]),\n",
       " 'split0_train_score': array([-1.17530571, -1.02375019, -0.77030834, -0.75376261, -0.67179203,\n",
       "        -0.67655334, -0.65973286, -0.66119217, -0.65854029, -0.65871696,\n",
       "        -0.65844751, -0.65841134, -0.65843061, -0.65838378]),\n",
       " 'split1_test_score': array([-1.18109743, -1.03204185, -0.78308089, -0.76713379, -0.68952977,\n",
       "        -0.69433003, -0.68055113, -0.68182075, -0.68034458, -0.68026928,\n",
       "        -0.6804275 , -0.68035155, -0.68044258, -0.68041573]),\n",
       " 'split1_train_score': array([-1.1733566 , -1.02168815, -0.76812863, -0.7515705 , -0.66956993,\n",
       "        -0.67431637, -0.65734783, -0.65879169, -0.65613608, -0.65629525,\n",
       "        -0.65602849, -0.65599192, -0.65601794, -0.65596114]),\n",
       " 'split2_test_score': array([-1.17464814, -1.02087093, -0.76889469, -0.75377286, -0.6763019 ,\n",
       "        -0.6810195 , -0.66728884, -0.66838086, -0.66686293, -0.66685688,\n",
       "        -0.66685781, -0.66680749, -0.66684838, -0.66681021]),\n",
       " 'split2_train_score': array([-1.17709402, -1.02534114, -0.77255438, -0.75591744, -0.67373834,\n",
       "        -0.67855149, -0.66150545, -0.66304678, -0.66028446, -0.66047969,\n",
       "        -0.66020441, -0.66016513, -0.660187  , -0.6601361 ]),\n",
       " 'split3_test_score': array([-1.17393875, -1.02329239, -0.76920902, -0.75495946, -0.6755592 ,\n",
       "        -0.68072181, -0.66529086, -0.6662869 , -0.66462057, -0.66442374,\n",
       "        -0.66471855, -0.66459388, -0.66473823, -0.66472994]),\n",
       " 'split3_train_score': array([-1.17745024, -1.02519383, -0.7729178 , -0.75607063, -0.67452557,\n",
       "        -0.67927736, -0.66250183, -0.66405059, -0.66129096, -0.66149476,\n",
       "        -0.6611969 , -0.66116426, -0.66117693, -0.661135  ]),\n",
       " 'split4_test_score': array([-1.1754269 , -1.02923794, -0.77566874, -0.76227154, -0.68334903,\n",
       "        -0.68809482, -0.67287071, -0.67439387, -0.6720142 , -0.67221916,\n",
       "        -0.67195284, -0.67192415, -0.67194275, -0.67189795]),\n",
       " 'split4_train_score': array([-1.17665304, -1.02351792, -0.77073484, -0.7543387 , -0.67310812,\n",
       "        -0.6777052 , -0.66106185, -0.66249963, -0.65987266, -0.66003555,\n",
       "        -0.65977365, -0.6597352 , -0.65975848, -0.65970716]),\n",
       " 'std_fit_time': array([ 0.06163599,  0.21278666,  0.17576173,  0.19103943,  0.63689327,\n",
       "         0.16522006,  1.28389909,  0.39634501,  1.2119076 ,  0.28931154,\n",
       "         1.0634835 ,  0.75958729,  1.87763521,  0.44746167]),\n",
       " 'std_score_time': array([ 0.0028886 ,  0.00202515,  0.00075178,  0.00303837,  0.00056298,\n",
       "         0.00181578,  0.00110211,  0.00240702,  0.00190724,  0.00146026,\n",
       "         0.00112442,  0.00192697,  0.00086231,  0.00133167]),\n",
       " 'std_test_score': array([ 0.00297272,  0.00401494,  0.00544041,  0.00506934,  0.0053201 ,\n",
       "         0.00531736,  0.00557861,  0.00564986,  0.00571081,  0.00572856,\n",
       "         0.00571145,  0.00572029,  0.00571135,  0.00570784]),\n",
       " 'std_train_score': array([ 0.00149614,  0.00132752,  0.00172408,  0.00164235,  0.00173677,\n",
       "         0.0017377 ,  0.00177955,  0.00181444,  0.00177879,  0.0017924 ,\n",
       "         0.00178471,  0.00178512,  0.00178185,  0.00178564])}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# view the complete results (list of named tuples)\n",
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.671833302852\n",
      "{'penalty': 'l2', 'C': 100}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果最佳值在候选参数的边缘，最好再尝试更大的候选参数或更小的候选参数，直到找到拐点。\n",
    "l2, c=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VOX1+PHPyb6xBBKSsC+CIEtA\nIqAgiIDgUhUV1Iq7Un/VVrt9a7W1rlVrtbZubFWxLmzKIm4oFVEUFFA2UUCQnSRkIQkJySzn98dM\nMEASZpJMZpKc9+s1r9x757l3zk10Ds+5z32uqCrGGGOMr8KCHYAxxpiGxRKHMcYYv1jiMMYY4xdL\nHMYYY/xiicMYY4xfLHEYY4zxiyUOY4wxfrHEYYwxxi+WOIwxxvglItgBBEJSUpJ27tw52GEYY0yD\nsWbNmoOqmuxL20aZODp37szq1auDHYYxxjQYIrLT17ZWqjLGGOMXSxzGGGP8YonDGGOMXyxxGGOM\n8YslDmOMMX6xxGGMMcYvljiMMcb4xRKHMcYYvzTKGwCNCRkuJxzOhsNZUJQNjsOgbs/L7f5pWV0V\nlt3gdoEqqBt1OVCnE5wu1OlEHQ7U6QKXC3V6ltXl9GxzOr3rbk9blwtcbs82pxN1uT3rLm+bo++7\nvevl77tRtxt16U/b3OpZd7tx5R8CICyhuec8VU889/JNqmhl2ytZ9xxGK297QrsKG6tqd8yCnvDZ\nWloKgERFnxB+favsV+jX/mWlhEUqXb/YVjcBVcMShzH+cjk8yaAoy/szs8JyFlqQieNAFo7sHJy5\nxTiKw3CUhOMsDsdVJqhbPF8SbvHkCfX8xLv9+PdB6u/cBCQMJEzA+1PCy9cFCRckTNBST7FCIyru\nWOEwUvn28k0/tavw/jFNqzjeSY8rx33+cfsc9/muvBIAIlonUD9O8resxZ/adbAECa/5/v6wxGEM\ngLPM2yvIqpAUvL2EosyfkkJRFs68QziLw3EcDsdR7EkIjuJwHCWROEoicBWXHzTK+4Lw5vGehBMR\nTsyppyIREUhEBEREIhHh3vVIiIw8+p5n3fNToqKQ8AgksrxNlGd7ZIU2kRFIeDhEVFiPiIDw8GPW\nPdsijln3xBKBhFn12pycJQ7TeDlLKySArGOTwTE/s+BIPqrgLhNPEvC+nKWxOEpjcZRE4ixSHAVx\n4I495mMkNobItDQiO7Ulpm0aEampRKamEXl0OZWw2NgqgjSm4bHEYRoWR0k1vYLjkkLpoWN2dTu9\nScHRDKezOY7SOBxHYnAWdcRxKA1HfjFa6jj28yIjiUxJITI1ldi2aTRPTSMyLZWItDRPskhNJax5\nc0/J5SQGv3Q5AKtufLPOfh3BMuilywBYdUPDPpfBL3v/JjU4D1/+5vWpPv/7Eq3tFZkQlJGRoTY7\nbiPxzBlQdADikz2Joqyw0mYa1QKHJuF0tcRRFoujJApnETgKnTjyinEeLMBVWHTsTiJEJCUdkwQi\n0lKJTGvrSQ6pqUQkJVVbvlFVikqd5Bc7OFTiIK+4jPxiB/klDg4Vl5FzuISswwfJKc3m66x1gJPU\nhCQU99EXuFF1o+JCVUF+WkcVrbiOevdx4bnk7N0ft2e/iuu4PW3E5W2nR9uV74co4D1u+XbxtKVi\n2/Jt3nWRxve9EapUfU9Q6opn081f1OhzRGSNqmb40tZ6HCY0lRbCO7+HnC1oRByuhF44ojNwFEfi\nLBYcBS4cecU4cgpwZufgzD4IWgKUHD1EWIsWRKamEtmpB7FDfiofeRJEGpFt2iBRnmsQFROA54u/\njPz9DvK37yb/cBn5Jd7txZ7lvOJiDpXlUuDIQcMLkIhCJMLzMyzip/WwiMNH44lq4/mZ69cvIgyp\n4hVGOCJh3jbiWS9/X8KQ8nUJQ4gk7Oiyd385tn358cKOf9+7rXw9TMJYn/k9ijAg9bTa/qWD6usD\n3wLQP8Xf86hJ4vR/H/Vjn3WZ3yH1NJDCEocJPfu+gXk34di7k71fdqTkgAt07TFNJMZ7XSEtlehT\nTzu6HJGaSlmrZIqat+YQkWSVlP30he/tCeTtKOPQpv3kl+w6ZrvLrSBOJLwQiSwgrEIyiIoqJCKq\niLDIAtyxBbjiPL2XmAoxhRFGi6jWtIppTUp8d1Lj25Ca0Ibk2GTaxLXhdx8+DoSxYMIUwiWcMAkj\nXMI9y2E/LZe/FyZhIVcOKVdeFnn18r8EOZLaKT+P165o2OcBP51LfbBSlQkdqrBqCu737iN3WxIH\nN0bhcLr5qudQ+p49gIJmrclLSORgTEuyJIb8Iw4OlSeD4rKjyy53Jf9NiwOJKCQu5jDx8cXExBR5\nEkFEIa6wQzglnyPufI64TyyFhUs4SbFJJMcmkxyXXOXPxOhEwsOqHg954/s3AvDSuJfq7FdmTF3x\np1RlicOEhsM56IJfUrh0GVmbUnAcciDDR3Jzs1PZG3nKCc0ToiNoERtJ8zglPq6YmJjDREYVIhGF\nuMMO4eAQRzSPw848DjkOUuQ4MSFESARJcd6EUE1SSIxJJExsmKpp3Owah2lYfvyMkqm3kvmZg5Ls\nVkT36MLGW2/k3h+jcchBmqXM4ar+Z1HqzqPQmUuBI5eDJdlkl2Szp/xiuYujlzciwiKOJoMOLbuS\nFDuINnFtTkgKLaNbWkIwpgYscZjgcTlxvv0gWdNmcmh7HOEtWxN796+4t7QrK37I4rSeX7Fb3wJx\nMeuHtUSERdAmtg1JcUl0bdGVQamehJAUm3TMzxbRLSwhGBNAljhMULizd5D752vIWZGDW+Npdd3P\nWTViAvd+uB133Me067OU3c5smkc3p218W6afN50W0S1C9mKxMU1JUBKHiLQCZgOdgR+Biaqad1yb\nkcA/K2zqCVylqgvqKUwTAKpK4cy/k/XcizgKw0gY2JPYv/yDv64t5P3/vUdipyWUhu2mXfNePJXx\nKIPTBgc7ZGPMcYLV47gbWKqqj4nI3d71P1ZsoKofA/3haKLZBiyp70BN3TmyaQOZf/wlxdsOEtUq\nkg5P3ceaU0byuwXvUpywkLiOW2kd35Y7T3+ccV3GWbnJmBAVrMRxCXCOd3kmsIzjEsdxrgDeU9Xi\natqYEOXMzSX77w+Tv/A9wiPdpFzen5g/Tue+pWtY/N7viGzzDc0jE/hl/z9wVc+riAqPCnbIxphq\nBCtxpKjqfgBV3S8ibU7S/irgqeoaiMhkYDJAx44d6yRIUztaVkbu669z8N9P4y45QmJPF8n3PsrK\nVoP5zWv3URzzCdEthUm9rmdy+i20iG4R7JCNMT4IWOIQkY+A1EreutfP46QBfYEPqmunqtOAaeC5\nj8OfzzB1S1Up+uQTsh59lLKdu4hPPULK1d1w3jiNyZ++xar1f0dij3BWynk8cPbvSEtIC3bIxhg/\nBCxxqOroqt4TkUwRSfP2NtKArGoONRGYr6qOatqYEFG6bRuZjz3O4c8+I6ql0H54HnFX/ZpXErvz\n77dvwhmWQ1JkX/4x+l4y0noHO1xjTA0Eq1S1CLgeeMz7c2E1ba8G/lQfQZmac+Xnk/3sc+S98QZh\nUeGkDCwiMT2BlWMe5i/fLyZz9xvgbMvNvR7lrmEXBTtcY0wtBCtxPAbMEZGbgV3ABAARyQBuU9Vb\nvOudgQ7AJ8EJ05yMOp3kzZrNwWeewVVYSMvTE0nusIkf+ozg9y0T+Wr9v3A7WtIl/BamT7yVtBZx\nwQ7ZGFNLQUkcqpoDjKpk+2rglgrrPwLt6i8y44+iz1aQ+dijlG37gbj+PUnpdoj8+K3c32s0iwq+\nRzNjcOVdyB/OvJkbzjzFbt4zppGwO8eN30p37CDr8b9TtGwZkR060P4X56AFs5mS2oH/xrXDUbCN\n0pyz6R59Cf++7iy6JicEO2RjTB2yxGF85ioo4ODzL5D72muERUXR5o5bSYhewtzCt5napRP56iSi\n6HSK94/iV8MHc/vIbkSE2018xjQ2ljjMSanLRf7ceWT/61+48vNpcfllJF3Ym/99+ReeDotib+tE\n2oT35PCWkXRu1oOXb+1PeoeWwQ7bGBMgljhMtQ6vXEXmo49S+v33xGYMJOX/fs/GHc/y2w1z2NQy\nlk6xHUjM/Bk/7O3A9Wd25u7zexEbVfXDjIwxDZ8lDlOpst27yfr73yn88CMi27al3dP/JLNHJL9f\ndiPLw52kxrbg3BY38e6qDiTGxTDzpnRG9EgOdtjGmHpgicMcw1V0mJypU8l9+WWIjCT5rjtxTriA\nJz+/hwXLvyFe4NaU8/h4x8Us3HSYC/ul8cilfWgZZ/NLGdNUWOIwAKjbzaH5C8h6+p+4sg/S4pJL\niP/VL5i5/03+u/hnONXFNTSjfadHeHhZGeFhpfzrqv5cnN7Whtka08RY4jAUr1lD5iN/48i33xLb\nvz+t//0vFsVsZurya8hzFHJ+UTG3dp7A41mX8vySHIae0ponrkinbcvYYIdujAkCSxxNmGPfPrL+\n8Q8K3n2PiNRU0p74O6v6RPKvr//M7sLdDDpSym9Lwjnc/2mu/DSGw6V53HfRadxwVmfCwqyXYUxT\nZYmjCXIXF5MzYwY5/3kRREi6/XZ2XtSf+799gfXL13MK0Tx/IIshbYfzcPNfMfP9Qvq0i+GfE/vT\nPaVZsMM3xgSZJY4mRN1uChYvJuvJp3BmZtL8wgs5MnkCD+59lWXLp9ImqiUPFrq4OG8nu0+/m5Eb\n+rPvUCF3jDyFX4/qTlSE3cxnjLHE0WSUrFvHgb/9jSPr1hPTpw/xj93HdD7lrVWTiYuI484W6Vyz\n/l2iW3bm5V7TeeizSDq2CmPubWcxsFNisMM3xoQQSxyNnCMzk6wnn6Rg0dtEJCfT6uH7mdsli1e+\nuxuH28HPu1zM5G1fkrjlbQ71uIKfZ07g69Uurh7UkT9f2Iv4aPtPxBhzLPtWaKTcR46Q8+KL5Eyf\nAS4XiZNvYfnIZJ7b8jy5G3MZ13kcv27Rjw4f/AV1lvFxrwf5xfoeNI+N4D/XD2BUr5Rgn4IxJkRZ\n4miEHPv388PYcWhZGc3GnsfWn5/J7/e9ys71OxmYMpBnRzxF36/nwMd3UJbch//jLhZ8HcfY3sn8\nbXxfWidEB/sUjDEhzBJHI3TwhSm4y8pY3zOaxWNzWPfdI3Rr0Y1nz32W4TFtkTdvggPr2dplEhO3\nn4+DSP4xoTeXn97ObuYzxpyUJY5GxrF3L/lvvcmyM2J4YbST5KJ93H/m/VxyyiVErJ8L71yNOyKa\nF1If5onNXRnUpRVPTkinQyt7Mp8xxjeWOBqZg9Om41ZlziAHbePbMf+S+cS5XbDgdlg/i7zkM7gm\n91a27W7OPRf04OZhXQm3m/mMMX6wxNGIOPbtI//NN/lkQCSu5Ga0TWhLXPYWmHcTmreDpSk3MXnn\nufRIbcHCW/rTK615sEM2xjRAljgakYPTpuFWN3MGKdPHTKPn1mXwnzGURbfid9EPsXhXFyaP6Mpv\nx/QgOsKemWGMqRlLHI2EY/9+8ue9ySf9I+jfezg9P3gAtrzHtsSzmXhgErEt2jDr1nQGd20d7FCN\nMQ2cJY5G4mhvY7AyNe4U2PIKU50X8ej+q7liYAf++rPTaBYTGewwjTGNgCWORsBx4AD5895keXoE\np/cbRfvvP2WftuYJ98+ZMmkg4/qkBjtEY0wjYrPWNQI506bjdruYM8TFbT0nEb1zGe+6BtGnbQtL\nGsaYOmc9jgbOceAAeXPnsjw9nNP7jaF75hZEHSyLGEp0pF0AN8bUPUscDVzO9Bm43S7mDQ5jSvpt\nFC34E4XaivPPv4hrhnQJdnjGmEbISlUNmCMzk7w5c/i0XzgD+o+le2wKsbs+4QPXIMb1aRvs8Iwx\njZT1OBqw8t7G3CFhTEv/f+iW94nQMnalnWcTFRpjAsZ6HA2UIzPL09voG87A/ufTrWU3CtfOY7+2\novvAUcEOzxjTiAUtcYhIKxH5UES2en9W+pg5Efm7iGwSkc0i8m+x6VsByJkxA7fTwbwhym3pt0Fp\nIXE7P+YD9yDG9kkLdnjGmEYsmD2Ou4GlqtodWOpdP4aInAUMBfoBfYAzgBH1GWQocmRlkTdnNp/2\ni2Dg6RfQtWVX9HtvmSrVylTGmMAKZuK4BJjpXZ4JXFpJGwVigCggGogEMusluhCWM2MGboeDN8/E\n09sACtfOI1NbcoqVqYwxARbMxJGiqvsBvD/bHN9AVb8APgb2e18fqOrmyg4mIpNFZLWIrM7Ozg5g\n2MHlyMoib/ZsPusTTsbpF9KlRRcoLSLWylTGmHoS0FFVIvIRUNmty/f6uP8pQC+gvXfThyIyXFWX\nH99WVacB0wAyMjK0ZhGHvtz/vOjpbZwVwbR+vwBAt3xApJay08pUxph6ENDEoaqjq3pPRDJFJE1V\n94tIGpBVSbPxwEpVLfLu8x4wBDghcTQFzuxscme9wYo+4ZyR8TM6t+gMQMHauZRqS04ZWOWv2xhj\n6kwwS1WLgOu9y9cDCytpswsYISIRIhKJ58J4paWqpiDnaG8jjF94exuUHSZu5/943z2I8+ymP2NM\nPQhm4ngMGCMiW4Ex3nVEJENEZnjbzAN+ADYA64B1qvp2MIINNufBg57eRm9Pb6Nj844A6JYlRLpL\n2Z0yxspUxph6EbQ7x1U1BzhhCJCqrgZu8S67gF/Uc2ghKec/L6KlZbw1NILp/X76lRSsnUuZtqDb\nGWOCGJ0xpimxO8cbAGdODrmvv85nfcIYdMaldGjewfNGWTGxPy7lA/cZnNenXXCDNMY0GZY4GoCc\n/7yIlpUx/6xwJvebfHS7bl1ClPsIO1POo1V8VBAjNMY0JTbJYYgr722s6B3G4MHjad+s/dH3CtbM\nxaHN6ZZhZSpjTP2xHkeIy33pJbS0lPlDw7m1360/veEtUy1xn8F5fdtXfQBjjKljljhCmDM3l5xX\nX+Pz08IYPOQy2iX8dB1Dt31IlLuEnalWpjLG1C8rVYUwT2/jCPOHRTGj7+Rj3itYMw+nNqPLwPOC\nFJ0xpqmyHkeIcublkfPqq3x+WhhDhlxBWkKFOagcJcTu+JAl7kFWpjLG1DtLHCEq98WX0CNHWDAs\n8thrG4Bu9ZSpdqWOtjKVMabeWakqBDnz8sh57VW+6BXGkDOvIDX+2HkiD615E7cm0GnguCBFaIxp\nyqzHEYJyX3oZLSlhwbBIbul7y7FvOo4Q++MSPrTRVMaYILHEEWI81zb+y8qeYZw5dOIJvQ3d9hHR\nrmJ+TB1jZSpjTFBYqSrE5L4809PbODuG6X1uPuH9Q2vmoZpAZytTGWOCxHocIcSVn39MbyMlPuXY\nBo4jxO74gA/dGYzp2yE4QRpjmjyfEoeIDBWReO/yJBF5SkQ6BTa0pidn5kw4XMyis6O5ue+JvQ39\n4X+eMlWKlamMMcHja4/jBaBYRNKB/wN2Aq8ELKomyJWfT84rr3h6G8OupE3cCY9gJ3/1XPI1nk4Z\n5wchQmOM8fA1cThVVYFLgH+p6r+AZoELq+nJfeUVOFzMwuHR3NTnphMbOEuJszKVMSYE+HpxvFBE\n/gRMAoaLSDgQGbiwmhbXoUMcnDmTVacKZ519FclxySe08ZSpDrPTylTGmCDztcdxJVAK3KyqB4B2\nwBMBi6qJyZ3p6W0sGh5TeW8DyF89j0MaR8eMC+o5OmOMOZbPPQ48JSqXiPQAegJvBC6spsNVUMDB\nV7y9jeE/Jyk26cRGzjJit3/AO1amMsaEAF97HMuBaBFpBywFbgReDlRQTUnuzFeg6DBvD4/lxj43\nVtpGt39MjKuQHSljSLQylTEmyHxNHKKqxcBlwDOqOh7oHbiwmgZXQQEHZ77Mlz2EoSOuoXVs60rb\n5a+eS4HG0cnKVMaYEOBz4hCRM4FrgHe828IDE1LTkfvf/3p6GyNiuaHPDZU3cpYR+8P7LHUPZHTf\njvUanzHGVMbXxHEX8CdgvqpuEpGuwMeBC6vxcxUWcvDllzy9jXMm0SqmVaXtdMcn3jLVaCtTGWNC\ngk8Xx1X1E+ATEWkmIgmquh34dWBDa9xy//tfKDzM4ivjmdb7hirb5X81l3CNpUPGRfUXnDHGVMPX\nKUf6isjXwEbgWxFZIyJ2jaOGXIWFHHzpRb7qLgw791oSYxKraOggZvt7VqYyxoQUX0tVU4Hfqmon\nVe0I/A6YHriwGre8V1+FwsO8c0481592fZXtdPsnxDoL2GmjqYwxIcTX+zjiVfXoNQ1VXVY+6aHx\nj6uoiOwXX2T1KcLQkdfSMqZllW3zVs8lUmNpl3FhPUZojDHV87XHsV1E/iIinb2vPwM7AhlYY+Xp\nbRR5ehu9q+5t4HIQ+8O7/M99OqP72kTExpjQ4WviuAlIBt4C5nuXK79bzQci0kpEPhSRrd6flRb5\nReRxEdnofV1Z088LFeW9jTWnCMNGXU+L6BZVttUdnxLrLLCb/owxIcenxKGqear6a1U9XVUHqOqd\nqppXi8+9G1iqqt3x3Il+9/ENRORC4HSgPzAY+IOINK/FZwZd3quvQUEh75yTwLWnXVt926/mUKQx\ntLcylTEmxFR7jUNE3ga0qvdV9eIafu4lwDne5ZnAMuCPx7U5DfhEVZ2AU0TWAeOAOTX8zKByFR0m\n+8UZrOkmnD36hmp7G7icxPzwHkvdAxjVt3O9xWiMMb442cXxfwToc1NUdT+Aqu4XkROfWgTrgL+K\nyFNAHDAS+DZA8QRc3uuvQ0ER705oxpTTJlXbVn/8lDhnPjtSxvAzK1MZY0JMtYnDe+NfjYjIR0Bq\nJW/d68v+qrpERM4APgeygS8AZzWfNxmYDNCxY2jd8+A+fJis/0zn627C2efdSPOo6ituuV/NJUaj\naZdR0w6dMcYEjk/DcUVkAyeWrA4Bq4GHVTXn+H1UdXQ1x8sUkTRvbyMNyKqsnao+Ajzi3ed1YGtV\nx1TVacA0gIyMjCrLa8GQ+/rryKFCT2+jV/W9DVxOYra9y8fu0xnVz0ZTGWNCj6+jqt7DM7nhNd7X\n28CnwAFqNr36IqB8LOr1wMLjG4hIuIi09i73A/oBS2rwWUHlPnyYrBnT+bqrMHzMzTSLqv6Ju7rz\nM+KdeexIGUPLOCtTGWNCj683AA5V1aEV1jeIyApVHSoiJ/kndKUeA+aIyM3ALmACgIhkALep6i14\nHk37qYgAFACTvBfKG5S8N95ADhXy3hXNeaHXNSdtn/vlXGI1mrYZP6uH6Iwxxn++Jo4EERmsqqsA\nRGQQkOB9z+8vc29pa1Ql21cDt3iXj+AZWdVguYuLyZwxjXVdhLPH3kxCVMJJdnARs+0dlrn7M6pf\n53qJ0Rhj/OVr4rgFeFFEEgDB0wO42TvtyKOBCq6hy3vjDSS/kPcub87zPX9+0va6cwXxzjy2p4zh\nAitTGWNClK/Tqn8F9BWRFnieBphf4e0GeV9FoLmLi8mc7ultDB9368l7G0DOl3OI1yjSBtpoKmNM\n6PJ1WvUW3vsplgIficiT3iRiqpA3azaSX8D7I5tzdc+rT76Dt0z1ifZnVHqXwAdojDE15OuoqheB\nQmCi91UAvBSooBo6d0kJmdOnsK6zMPz8W4mPPPlEwrrzcxIcufyQbKOpjDGhzddrHN1U9fIK6w+I\nyDeBCKgxyJs1G8kr4IPxLXnOl94GkPPVXBI0krQzLglwdMYYUzu+9jhKRGRY+YqIDAVKAhNSw+Yu\nKSFz2gus7ywMv2AycZFxPuzkJnrrOyzTAZzbz8pUxpjQ5muP4/8BM8svjgO5wA2BCqohy5v9U2/j\n2VN9mwled31BM8dBtidPZpyVqYwxIc7XUVXfAOnl05qrakFAo2qg3EeOkDltKhs7CSMuvM233gae\n0VTNNJJUK1MZYxqAk02r/tsqtgOgqk8FIKYGK3/2bCQ3nyWXJPLMqRN928lbpvpE0zm3X9fABmiM\nMXXgZD2O6idWMke5jxxh/9QX+LaTMPyi24iNiPVpP929imaObLa3uYXzrExljGkATjat+gP1FUhD\nlz9nDmG5h/jwklb829feBnDwyzk010jaZFwawOiMMabu+Dqq6igRWRuIQBoyd2kp+6e+wMaOwvCf\n/T9iImJ83NFN9Ja3WW5lKmNMA+J34sAzqspUkDdnDmE5+Xw0KpErelzh836650uaO7LZnjzKbvoz\nxjQYNUkc79R5FA2Yu7SUA1OfZ1NHGH7xL33vbQAHV82hVCOsTGWMaVD8Thyq+udABNJQ5c+dS9jB\nfD4a1dqv3oZnNNXbfKb9ODe9e+ACNMaYOubrJIeFIlJw3Gu3iMwXkSZbnHeXlrL/hef4tgOMuPiX\nRIdH+7yv7l1N87IsfkgeTYu4yABGaYwxdcvXO8efAvYBr+O5xnEVkAp8j2cCxHMCEVyoy587j7Cc\nfJZenMTT/vQ2gOxVs2mp4VamMsY0OL6Wqsap6lRVLVTVAlWdBlygqrOBxADGF7LcZWXsn/Icm9vD\niItvJyrcj4vbqkRvWcwK7cfI9B6BC9IYYwLA18ThFpGJIhLmfVW8UUEDEVioy587j7CDeSwdk8T4\nHpf5ta/uXUOLsgNsszKVMaYB8jVxXANcC2QBmd7lSSISC9wRoNhClrusjH1TnuW79jDikjv8623g\nKVOVaTjJGeMDFKExxgSOr5Mcbgd+VsXbn9VdOA1D/rw3Cc/O4383tOGp7v71NlAl6vtFfK59rUxl\njGmQfB1V1UNElorIRu96PxFpksNy3WVl7HvhGb5rByPG30FkuH+lJt27lpZlB/gheZSVqYwxDZKv\nparpwJ8AB4CqrsczsqrJyX/rLcKz81g2pg2XnOL/iKjsVbNxaDitB/rZUzHGmBDha+KIU9Uvj9vm\nrOtgQp2WlbHv+Wf4vh0MH/8rv3sbqBK55W0+1z6M7H9qYII0xpgA8zVxHBSRbnhHUInIFcD+gEUV\novLnzyc8K5ePx7Th4u7+P3RJ931DYuk+u+nPGNOg+XoD4O3ANKCniOwFduAZadVkaFkZe5//Nz+0\nhXMuu5PIMP+/+LNWzaaVhtPKylTGmAbM1x7HXuAl4BFgFvAhcH2gggpF+fMXEJ6Zy7IxbbjolKoG\nmFXDO5pqpfa2MpUxpkHzNXEsxDMc14Fn6pEi4HCgggo1WlbGnuf/zdY0GHHFXTXqbej+b0gs3cu2\nJBtNZYxp2HwtVbVX1XEBjSTuGnYkAAAVrklEQVSE5S9YSERmDp9cn8rj3WrQ2wCyVs2ltYbROuPy\nOo7OGGPql689js9FpG9dfaiITBCRTSLiFpGMatqNE5HvRWSbiNxdV5/vD3U42PPcv9iWBiMm3EVE\nmK+5tuJBlMjvFrJSezNiQM+6D9IYY+qRr4ljGLDG+yW+XkQ2iMj6WnzuRuAyYHlVDUQkHHgOOB84\nDbhaRE6rxWfWSP5Cb2/jvFQu6HphjY6hB9bTqnSP56a/WCtTGWMaNl//+Xx+XX6oqm4GEKn2KbSD\ngG3e6U4QkVnAJcC3dRlLddThYM+zT/NjKoyY8Jua9TaAzJWzSdIwEgdamcoY0/D5OlfVzkAHUol2\nwO4K63uAwfUZQP6iRUQcyOHT69J4tIa9DU+ZahGr9DRGDOhVtwEaY0wQ1Oyf0D4QkY/wPOzpePeq\n6kJfDlHJtiqncBeRycBkgI4dO/oUY3XU4WD3s/9kVyqMuPK3hIeF1+w4mRtpXbqbbcmXM9TKVMaY\nRiBgiUNVR9fyEHuADhXW2+MZClzV503Dc5MiGRkZtX5GSP7bi4jcn8Nn17XlkS41r9RlfjGbZBW7\n6c8Y02j4enE8GL4CuotIFxGJwjOp4qL6+GB1Otn9zNNsT4URV/2uxr0NVIn4fiFfai+GD6j36/rG\nGBMQQUkcIjJeRPYAZwLviMgH3u1tReRdAFV14nlI1AfAZmCOqm6qj/jyFi0icv9BPjuvHed1Hlvj\n42jmJpKO7GJr0mgbTWWMaTQCVqqqjqrOB+ZXsn0fcEGF9XeBd+sxNNTpZM+z/2R3Coy4uha9DeDA\nytm0UaGV3fRnjGlEQrlUFRT5by8ict9BVoytXW8DIOK7RXylvTh7QO86is4YY4LPEkcF6nSy65l/\nsiMFzrn6D4RJzX89mvktyUd+9MxNZWUqY0wjYomjgrzFbxO17yCfj23P6M5janWsA1/Mwq1CS7vp\nzxjTyATlGkcoUpeLbQ/eS04bOOfn/1er3gZ4ylSr9VTOPr1PHUVojDGhwXocXo7iw3zeO5wF58Ry\nbqdRtTqWZm0m+cgOG01ljGmUrMfh5YyJ4O2L29A8qnmtexsHvphNipWpjAkKh8PBnj17OHLkSLBD\nCUkxMTG0b9+eyMia/6PWEodXXGQcSycsrZNjhX+3kLXag2Gn19lM9MYYH+3Zs4dmzZrRuXPnk02k\n2uSoKjk5OezZs4cuXbrU+DhWqqpjmv09bUq2s8XKVMYExZEjR2jdurUljUqICK1bt651b8wSRx3b\n/8UsAFqcbmUqY4LF36Rx5dQvuHLqFwGKJrTURUK1xFHHwjcvYo37VIYN7BfsUIwxQZKQkHB0edy4\ncbRs2ZKLLrqo0ra33347/fv357TTTiM2Npb+/fvTv39/5s2b59dnrl27lvfff79WcfvKrnHUIXf2\nVlJKtvG/pF8y0MpUxhjgD3/4A8XFxUydOrXS95977jkAfvzxRy666CK++eabGn3O2rVr2bhxI+PG\njatxrL6yHkcdOuAtU9loKmNMuVGjRtGsWbMa7bt161bGjh3LwIEDGT58OFu2bAFg1qxZ9OnTh/T0\ndEaOHElJSQkPPvggr732Wo16K/6yHkcdCtu8kLXu7gwdmB7sUIwxwANvb+LbfQUnbfftfk8bX65z\nnNa2OX/9Wf3MPzd58mRmzJhBt27dWLFiBXfccQdLlizhgQceYNmyZaSkpJCfn09sbCz33XcfGzdu\n5Omnnw54XJY46og7exupJVv5JOk2To+xMpUxpnby8/NZuXIll1/+UwXD6XQCMHToUK677jomTJjA\nZZfV/0PiLHHUkf0rZ9EOaG6jqYwJGb72DMp7GrN/cWYgw/GLqpKUlFTpNY/p06ezatUqFi9eTHp6\nOuvXr6/X2OwaRx0J27yQb9ynMDRjQLBDMcY0AomJiaSlpTF/vufRRW63m3Xr1gGwfft2hgwZwkMP\nPURiYiJ79+6lWbNmFBYW1ktsljjqgPvgdtKKt7AlaRTNrUxljKng7LPPZsKECSxdupT27dvzwQcf\n+LzvrFmzmDJlCunp6fTu3ZvFixcD8Jvf/Ia+ffvSt29fRo8eTZ8+fTj33HNZt24dAwYMsIvjDcG+\nL96gPdDs9CuCHYoxJgQUFRUdXf7000992qdz585s3LjxmG1du3atNNEsWrTohG3JycmsXr3az0hr\nxhJHHQjbvIh17m5WpjKmgQqlaxsNgZWqasmds4O2xd/xvZWpjDFNhCWOWtpnc1MZY5oYSxy1FPbt\nQja4u3JmxsBgh2KMMfXCEkcteMpUm/m+tZWpjDFNhyWOWtj3xRwAmg200VTGNGgvXeh5GZ9Y4qgF\n+XaBlamMMSeo72nV58+fzxNPPFHruH1lw3FryJ27k3bF37Ky9S30tTKVMaYKdTWtutPpJCKi8q/s\n8ePH102wPrIeRw3t/WI2AM1sCnVjTDVqM636sGHDuPfeexk+fDjPPvssCxcuZPDgwQwYMIDzzjuP\nrKwsAGbMmMFdd90FwKRJk7jzzjs566yz6Nq169EpS+qS9ThqSL5dwCbtzJkZZwQ7FGNMVd67Gw5s\nOHm7A95JAn25zpHaF85/rHZx+aGgoIDly5cDkJeXx8UXX4yIMGXKFJ588kkef/zxE/bJyspixYoV\nbNiwgYkTJ9Z5j8QSRw2483bR/vAmVrW6md5WpjLGBNBVV111dHnXrl1MnDiRAwcOUFpaSo8ePSrd\n59JLL0VE6NevH3v37q3zmIKSOERkAnA/0AsYpKqVTrAiIi8CFwFZqtqn/iKs3t4vZtMBaGY3/RkT\n2nztGZT3NG58J3Cx1FB8fPzR5dtvv5177rmHCy64gI8++ojHHqv8/KKjo48uq2qdxxSsaxwbgcuA\n5Sdp9zIQ+Afo+mvTAjZrJ848Y1CwIzHGNCGHDh2iXbt2qCozZ84MWhxBSRyqullVv/eh3XIgtx5C\n8pk7bzcdDm/ku1ajaGZlKmPMSdRmWvXj3X///YwfP54RI0aQkpJSh1H6x65x+Km8TJVgU6gbY6pQ\nV9Oqf/bZZ8esX3755cc8SrbcLbfccnT51VdfrTKWuhKwxCEiHwGplbx1r6ouDMDnTQYmA3Ts2LGu\nD3+UblrId9qRIVamMqbxCMFrG6EsYIlDVUcH6thVfN40YBpARkZG3V8NAtz5e+l4eD0LWt1ATytT\nGWOaKLsB0A97vFOoJwywMpUxpukKSuIQkfEisgc4E3hHRD7wbm8rIu9WaPcG8AVwqojsEZGbgxFv\nOd20kO+1A4MHDQlmGMYYE1RBuTiuqvOBE+6DV9V9wAUV1q+uz7iq4z60jw5F63m71XWcamUqY0wT\nZqUqH+3+fDZhKPFWpjKm0bnx/Ru58f0bgx1Gg2GJw0e6aQFbtT2DB9lD7Y0x1SufVv2bb77hzDPP\npHfv3vTr14/Zs2ef0LYuplUHWLt2Le+//36dxH8ydh+HD9yH9tOxaB2LE6+lu5WpjDE+iouL45VX\nXqF79+7s27ePgQMHMnbsWFq2bHm0ja/Tqp/M2rVr2bhxI+PGBX6yDetx+KC8TJVgc1MZY/zQo0cP\nunfvDkDbtm1p06YN2dnZPu+/detWxo4dy8CBAxk+fDhbtmwBYNasWfTp04f09HRGjhxJSUkJDz74\nIK+99lqNeiv+sh6HD9ybFrJN2zFo8LBgh2KM8cPjXz7Od7nfnbRdeRtfrnP0bNWTPw76o9+xfPnl\nl5SVldGtWzef95k8eTIzZsygW7durFixgjvuuIMlS5bwwAMPsGzZMlJSUsjPzyc2Npb77ruPjRs3\n8vTTT/sdm78scZyEuyCTjkXf8G7izzkl2n5dxhj/7d+/n2uvvZaZM2cSFuZboSc/P5+VK1ceM8WI\n0+kEYOjQoVx33XVMmDCByy67LCAxV8e+CU9i9+ez6YTbRlMZ0wD52jMo72m8NO6lOo+hoKCACy+8\nkIcffpghQ3y/B0xVSUpKqvSax/Tp01m1ahWLFy8mPT2d9evX12XIJ2XXOE7CvXE+27WtlamMMX4r\nKytj/PjxR3sH/khMTCQtLe3oo1/dbjfr1q0DYPv27QwZMoSHHnqIxMRE9u7dS7NmzSgsLKzzc6iM\nJY5quAuz6Fj0Dd8mnkuCjaYyxvhpzpw5LF++nJdffvnoMFt/Rk3NmjWLKVOmkJ6eTu/evVm8eDEA\nv/nNb+jbty99+/Zl9OjR9OnTh3PPPZd169YxYMAAuzgeTLs+n0Nn3MQPsNFUxhjflU9lPmnSJCZN\nmuTTPpVNq961a9dKn9+xaNGiE7YlJyezenWlD1Otc5Y4quHeOJ8dmsoZg88OdijGmAAKxLWNxsxK\nVVVwF2bTqXAtm61MZYwxx7DEUYVdn88mHDex/a1MZYwxFVniqIJr4wJ+1FTOGDIi2KEYY0xIscRR\nCXfRQToVruHbxJFWpjLGmONY4qjEzs/nEIGbOCtTGdMk7Lz2OnZee12ww2gwLHFUwrVhPru0DRlD\nzgl2KMaYBqi+p1WfP38+TzzxRJ3FfzI2HPc47qIcOheu5qPEiYyzMpUxphbqclp1p9NJRETlX9nj\nx4+v++CrYYnjODu/mEsX3MRYmcoYU0s9evQ4ulxxWvWKiaM6w4YNY8SIEXz66adcdtlldOnShb/9\n7W+UlZWRnJzMq6++Sps2bZgxY8bRmXEnTZpE69at+eqrrzhw4ABPPvlknScWSxzHcW6Yz25NJmPI\nyGCHYoyppQN/+xulm08+rfqR7zxtfLnOEd2rJ6n33ON3LDWZVh08kyQuX74cgLy8PC6++GJEhClT\npvDkk0/y+OOPn7BPVlYWK1asYMOGDUycONESRyC5D+fSpeArliZezlgrUxlj6khNplUvd9VVVx1d\n3rVrFxMnTuTAgQOUlpYe06Op6NJLL0VE6NevH3v37q1V7JWxxFHBjyvm0hUXMelWpjKmMfC1Z1De\n0+j031fqPIaaTqteLj4+/ujy7bffzj333MMFF1zARx99xGOPPVbpPtHR0UeXVdX/oE/CRlVV4No4\nnz2azMAzRwU7FGNMI1CbadUrc+jQIdq1a4eqMnPmzDqIsGYscXi5jxTR4dBqvtZT7KY/Y0ydqO20\n6se7//77GT9+PCNGjCAlJaUOI/WPBKIbE2wZGRnq7/TCRxwuJvxjIQlRwhu/q9+hbcaYurN582Z6\n9erl1z6BLFWFosp+RyKyRlUzfNnfrnF4xUSG8/af6v/ZvcaY4GsqCaOuWKnKGGOMXyxxGGOM8Ysl\nDmNMo9MYr93Wlbr43QQlcYjIBBHZJCJuEan0YoyIdBCRj0Vks7ftnfUdpzGm4YmJiSEnJ8eSRyVU\nlZycHGJiYmp1nGBdHN8IXAZMraaNE/idqq4VkWbAGhH5UFW/rZcIjTENUvv27dmzZw/Z2dnBDiUk\nxcTE0L59+1odIyiJQ1U3A4hIdW32A/u9y4UishloB1jiMMZUKTIyki5dugQ7jEatQVzjEJHOwABg\nVTVtJovIahFZbf/SMMaYwAlYj0NEPgJSK3nrXlVd6MdxEoA3gbtUtaCqdqo6DZgGnhsA/QzXGGOM\njwKWOFR1dG2PISKReJLGa6r6Vu2jMsYYU1she+e4eC6A/AfYrKpP+bPvmjVrDorIzhp+dBJwsIb7\nhprGci6N5TzAziUUNZbzgNqdSydfGwZlrioRGQ88AyQD+cA3qjpWRNoCM1T1AhEZBnwKbADc3l3v\nUdV3Axzbal/nawl1jeVcGst5gJ1LKGos5wH1dy7BGlU1H5hfyfZ9wAXe5c+AqoddGWOMCYoGMarK\nGGNM6LDEcaJpwQ6gDjWWc2ks5wF2LqGosZwH1NO5NMrncRhjjAkc63EYY4zxiyWOSojIQyKyXkS+\nEZEl3tFeDY6IPCEi33nPZb6ItAx2TDXly8SYoUxExonI9yKyTUTuDnY8tSEiL4pIlohsDHYstdGY\nJlIVkRgR+VJE1nnP5YGAfp6Vqk4kIs3L71IXkV8Dp6nqbUEOy28ich7wP1V1isjjAKr6xyCHVSMi\n0gvPsOypwO9V1b9nAweRiIQDW4AxwB7gK+Dqhjphp4gMB4qAV1S1T7DjqSkRSQPSKk6kClzaEP8u\n3vve4lW1yHvj9GfAnaq6MhCfZz2OShw3tUk80CCzq6ouUVWnd3UlULspMYNIVTer6vfBjqOGBgHb\nVHW7qpYBs4BLghxTjanqciA32HHUlqruV9W13uVCoHwi1QZHPYq8q5HeV8C+tyxxVEFEHhGR3cA1\nwH3BjqcO3AS8F+wgmqh2wO4K63tooF9QjZUvE6mGOhEJF5FvgCzgQ1UN2Lk02cQhIh+JyMZKXpcA\nqOq9qtoBeA24I7jRVu1k5+Ftcy+e55u8FrxIT86Xc2mgKruRtUH2YhsjXydSDXWq6lLV/ngqC4NE\nJGBlxJCdqyrQ/JiE8XXgHeCvAQynxk52HiJyPXARMEpD/IJWXUyMGaL2AB0qrLcH9gUpFlNBY5xI\nVVXzRWQZMA7PQ/PqXJPtcVRHRLpXWL0Y+C5YsdSGiIwD/ghcrKrFwY6nCfsK6C4iXUQkCrgKWBTk\nmJq82kykGmpEJLl81KSIxAKjCeD3lo2qqoSIvAmcimcUz07gNlXdG9yo/Cci24BoIMe7aWVDHB0G\nVU+MGdyofCciFwBPA+HAi6r6SJBDqjEReQM4B89MrJnAX1X1P0ENqgaCNZFqIIhIP2Amnv++woA5\nqvpgwD7PEocxxhh/WKnKGGOMXyxxGGOM8YslDmOMMX6xxGGMMcYvljiMMcb4xRKHMTUgIkUnb1Xt\n/vNEpKt3OUFEporID96ZTZeLyGARifIuN9kbdU1ossRhTD0Tkd5AuKpu926agWfSwO6q2hu4AUjy\nToi4FLgyKIEaUwVLHMbUgng84Z1Ta4OIXOndHiYiz3t7EItF5F0RucK72zXAQm+7bsBg4M+q6gbw\nzqL7jrftAm97Y0KGdYGNqZ3LgP5AOp47qb8SkeXAUKAz0Bdog2fK7he9+wwF3vAu98ZzF7yriuNv\nBM4ISOTG1JD1OIypnWHAG96ZSTOBT/B80Q8D5qqqW1UPAB9X2CcNyPbl4N6EUuZ90JAxIcEShzG1\nU9mU6dVtBygBYrzLm4B0Eanu/8Vo4EgNYjMmICxxGFM7y4ErvQ/RSQaGA1/ieXTn5d5rHSl4JgUs\ntxk4BUBVfwBWAw94Z2tFRLqXP4NERFoD2arqqK8TMuZkLHEYUzvzgfXAOuB/wP95S1Nv4nkOx0Y8\nz0lfBRzy7vMOxyaSW4BUYJuIbACm89PzOkYCDW62VtO42ey4xgSIiCSoapG31/AlMFRVD3ifl/Cx\nd72qi+Llx3gL+FMDft66aYRsVJUxgbPY+3CdKOAhb08EVS0Rkb/iee74rqp29j70aYElDRNqrMdh\njDHGL3aNwxhjjF8scRhjjPGLJQ5jjDF+scRhjDHGL5Y4jDHG+MUShzHGGL/8f2UPs7GhIRwqAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e019790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pd.DataFrame(grid.cv_results_).to_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#cvresult = pd.DataFrame.from_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#test_means = cv_results['mean_test_score']\n",
    "#test_stds = cv_results['std_test_score'] \n",
    "#train_means = cvresult['mean_train_score']\n",
    "#train_stds = cvresult['std_train_score'] \n",
    "\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的正确率（score）。可以看出在训练集上C越大（正则越少）的模型性能越好；但在测试集上当C=100时性能最好（L1正则和L2正则均是）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L1正则 --> 可选用saga优化求解器(0.19版本新功能)\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.08352368, -0.08351874, -0.08351932, -0.0835193 ],\n",
       "        [-0.08855243, -0.08870811, -0.08872645, -0.08872815],\n",
       "        [-0.08310835, -0.08322735, -0.08324098, -0.08324238],\n",
       "        [-0.0821809 , -0.08218971, -0.08219017, -0.08219025],\n",
       "        [-0.08079014, -0.08076733, -0.08076566, -0.08076557]]),\n",
       " 1: array([[-0.31682739, -0.31682268, -0.31682257, -0.31682187],\n",
       "        [-0.32032738, -0.32039225, -0.32039847, -0.32039874],\n",
       "        [-0.31523852, -0.31526591, -0.31526951, -0.31527038],\n",
       "        [-0.31668845, -0.31669211, -0.31669289, -0.31669223],\n",
       "        [-0.31923648, -0.3192882 , -0.31929246, -0.31929285]]),\n",
       " 2: array([[-0.26140663, -0.26141912, -0.26142148, -0.26142154],\n",
       "        [-0.261854  , -0.26188003, -0.26188282, -0.26188393],\n",
       "        [-0.26767802, -0.26772406, -0.26773324, -0.2677292 ],\n",
       "        [-0.26380728, -0.26382864, -0.26383156, -0.26383158],\n",
       "        [-0.26326535, -0.26328817, -0.26329084, -0.26329134]]),\n",
       " 3: array([[-0.13062825, -0.13080908, -0.13082942, -0.13083111],\n",
       "        [-0.12742622, -0.12747526, -0.12748394, -0.12748482],\n",
       "        [-0.12529098, -0.12529292, -0.12529459, -0.1252947 ],\n",
       "        [-0.12718137, -0.12721724, -0.12722167, -0.12722211],\n",
       "        [-0.13061306, -0.13074839, -0.13076616, -0.13076824]]),\n",
       " 4: array([[-0.01544939, -0.01577876, -0.01586219, -0.01587077],\n",
       "        [-0.01202258, -0.01223192, -0.01236506, -0.01238369],\n",
       "        [-0.01433804, -0.01462026, -0.01466695, -0.01467094],\n",
       "        [-0.01854087, -0.01919685, -0.01949191, -0.01952841],\n",
       "        [-0.01407623, -0.01390069, -0.01391826, -0.01392047]]),\n",
       " 5: array([[-0.11884352, -0.11891645, -0.11892368, -0.11892457],\n",
       "        [-0.11643966, -0.11660762, -0.11662221, -0.11662289],\n",
       "        [-0.10609816, -0.10607444, -0.10607096, -0.10607168],\n",
       "        [-0.10410981, -0.10414617, -0.10414875, -0.10414957],\n",
       "        [-0.10736375, -0.10739667, -0.10740061, -0.10740235]]),\n",
       " 6: array([[-0.10058127, -0.10063923, -0.10064592, -0.10064661],\n",
       "        [-0.11305482, -0.11313295, -0.11314162, -0.11314253],\n",
       "        [-0.10450561, -0.10452733, -0.10453016, -0.10453036],\n",
       "        [-0.10078705, -0.10081466, -0.1008181 , -0.10081853],\n",
       "        [-0.10068649, -0.10074311, -0.10075039, -0.10075115]]),\n",
       " 7: array([[-0.10177277, -0.10178626, -0.10178828, -0.10178923],\n",
       "        [-0.11022222, -0.11031416, -0.11032385, -0.11032495],\n",
       "        [-0.10008425, -0.10007941, -0.10007911, -0.1000778 ],\n",
       "        [-0.10345949, -0.10347857, -0.1034821 , -0.10348144],\n",
       "        [-0.10526816, -0.10530963, -0.10531493, -0.10531428]]),\n",
       " 8: array([[-0.09436476, -0.09453015, -0.09454934, -0.09455107],\n",
       "        [-0.08332469, -0.08332992, -0.08333145, -0.08333171],\n",
       "        [-0.09287026, -0.09289505, -0.09289854, -0.09289893],\n",
       "        [-0.09000692, -0.0901021 , -0.09011307, -0.09011411],\n",
       "        [-0.09095359, -0.09094999, -0.09095094, -0.09095088]])}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAERCAYAAABLmsECAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcVNWZ//HP03uzb80ijQICsgi4\ntFvct4grKKBkNC4/jZmMjpPJRBTHxGhMjDoTx0k0iYlbnERkcUFFiQsu0QgUIM0uLSjd7Pve9Pb8\n/qiLtk03XTRddau6v+/Xq17e5dxTz7GaeuqcOnWuuTsiIiKJkhZ2ACIi0rwo8YiISEIp8YiISEIp\n8YiISEIp8YiISEIp8YiISEIp8TSAmY02s4VmVmVmBXWUyTGzmWY2Lyh7b7Vzz5jZCjP7NHgcExy/\n2swKg8fHZja0EWI9w8zmmFmFmY061PpERA6VEk89zOwsM3umxuEFwBXABwe4dC9wjrsPBY4BhpnZ\nydXO3+7uxwSPT4NjK4Az3X0I8HPgiUZowkrgeuCvjVCXiMghywg7gFTk7osBzOxAZRzYGexmBo8D\n/lrX3T+utvsJkL9vx8yuAW4DsoAZwL+4e2UMsX4RXF9VX1kRkURQjyeOzCzdzD4F1gNvufuMaqd/\nEQypPWJm2bVcfiPwRlDPAOAq4FR3PwaoBK6Oc/giInGhHk8dzGwGkA20AjoECQTgDnefFksdQY/k\nGDNrB7xkZke7+wJgHLCWaO/lCeAO4L5qz3020cRzWnDoXOB4YFbQy8olmswwsz8Dx9Xy9I+7++Ox\nt1hEJDGUeOrg7idB9Dse4Hp3v/4Q6tpqZu8Bw4AF7r4mOLXXzJ4GfryvrJkNAf4EXOjum/YdBp51\n93G11H1tQ+MSEQmDhtrixMzygp4OZpYLnAcsCfa7Bf81YATRyQqY2eHAi8B33f2zatW9A4wys85B\nuQ5mdkSi2iIi0piUeBrAzC43sxLgFOB1M5sWHD/MzKYGxboB082sEJhF9Due14JzfzGz+cB8oBNw\nf3D8p0BH4PFgmnUEwN0XAXcDfwvqeyuoP5ZYTwhiHQ38wcwWHlLjRUQOkem2CCIikkjq8YiISEJp\nckEtOnXq5D179gw7DBGRlDJ79uyN7p5XXzklnlr07NmTSCQSdhgiIinFzL6MpZyG2kREJKGUeERE\nJKGUeEREJKGUeEREJKGUeEREJKGUeEREJKGUeEREJKH0Ox4RkSaqssrZU17JnrLgUV7J7rKKr4+V\nV7K7rJLS4L+7yyo5t39nhvZoF9e4lHhEREJSXln1jTf+aDKoYE9Z1TcSxO4gSeyfLCrYU17FnqDs\n7rJvlimrOPgbD3duna3EIyISBndnb0VVHT2Diq+TRbUexe79ehdfJ4hvJJigXEXVwS3SnGbQIiuD\nnMx0WmRFH/u22+RkkpuVTm6wn5OVTovMDHKz0sjNyvjqePUyuZnV9zPIzkgjLc3i9H/0a0o8IpKS\nqqqc0or9P+XX7DXslyBq7NfsNVRPHAeZF8hMt6/ezFsEb/a5Wem0zM6gY6vsWt7s9yWOjG8kkdxq\n5apfk5WeRnAX4pSmxCMiCVe8eTdzi7dG3+yDnkJpWc2EUDNZVFBa/vUQVGn5wQ8jZWekBT2FDHIy\n075KDu1yM+nWJqdaTyF486/eO6jWa8ip1mPYd01uZjqZ6ZqvFQslHhFJiNLySqYtXMuESDEfFW3a\n77wZ+w0HRd/s08hrnU1uZotaegrfHCrKzUojNzNjv55Cbmb0kYhhJKmfEo+IxI27U1iyjQmRYqbM\nW82O0gp6dMjlR+f34/yBXWiTm/lV7yI7o2kMI0n9lHhEpNFt2rmXl+auYmKkhKXrdpCdkcZFg7sx\nuiCfk3t1VM+jmVPiEZFGUVFZxQfLNjBhVgnvLFlHeaUztEc7fnH50Vw69DDa5GSGHaIkCSUeETkk\nyzfsZOLsEibPLmH9jr10bJnF9d/qyeiCHvTr0jrs8CQJKfGIyEHbtbeC1+evYWKkmFlfbCE9zTj7\nqDxGF/Tg7KM6k5Wh2V1SNyUeEYmJuzP7yy1MiBTzWuEadpdV0juvJXde2J8rju1O5zY5YYcoKUKJ\nR0QOaN32UibPKWFSpITlG3fRMiudS4ccxpUn5HPc4e01E00OmhKPiOynrKKKd5esY0KkhPeWrqfK\n4cSeHfjBWUdy0eButMzWW4c0nP56ROQrS9fuYEKkmJfmrmLzrjK6tMnmB2cdyajje9CrU8uww5Mm\nQolHpJnbtqecV+etZmKkmHkl28hMN84f2IXRBT04vU8nMrQMjDQyJR6RZqiqyvnH8k1MiBTz5oK1\n7K2oon/X1vz0koGMOLY7HVpmhR2iNGFKPCLNSMmW3UyaXcLESAmrtu6hTU4GVxb04MqCHhzdvY0m\nCkhCKPGINHH7FuecGCnho883AnDqkZ0YO+woLhjUlZzM9JAjlOZGiUekCXJ3FqzazoRIMa98uort\npRXkt8/lh+f2Y+Tx3clv3yLsEKUZi2viMbNhwKNAOvAnd/9VjfPZwJ+B44FNwFXu/kVwbhxwI1AJ\n3Obu0w5Up5ndCvwQOBLIc/eNwXELyl8E7Aaud/c5cWy2SGg27yrj5bmrmBApZsna6OKcFx7dlSsL\nenByby3OKckhbonHzNKBx4DzgRJglplNcfdF1YrdCGxx9z5mNgZ4ELjKzAYCY4BBwGHA22bWL7im\nrjo/Al4D3qsRyoVA3+BxEvC74L8iTUJFZRUfLtvIxNnFvLUoWJwzvy33j4guztk2V4tzSnKJZ4/n\nRKDI3ZcDmNl4YDhQPfEMB34WbE8Cfhv0UIYD4919L7DCzIqC+qirTnefGxyrGcdw4M/u7sAnZtbO\nzLq5+5pGba1Igq3YuIuJkWImzylh3fa9dGiZxbWn9GR0QT79u7YJOzyROsUz8XQHiqvtl7B/T+Or\nMu5eYWbbgI7B8U9qXNs92K6vzlji6A58I/GY2c3AzQCHH354PVWKhGPX3gqmzl/DxEgJM7/YTJrB\nWUd15t7L8jmnfxctzikpIZ6Jp7bBZI+xTF3Ha/tXVbPOhsSBuz8BPAFQUFBQX50iCePuzFm5hQmz\nSnitcDW7yirp3akldwzrzxXHdaeLFueUFBPPxFMC9Ki2nw+srqNMiZllAG2BzfVcW1+dDYlDJOms\n317Ki8FEgeUbdtEiK51LhnTjyoIeHH+EFueU1BXPxDML6GtmvYBVRCcL/FONMlOA64B/AKOAd93d\nzWwK8Fcz+zXRyQV9gZlEey/11VnTFODW4Pugk4Bt+n5HklV5ZRXvLlnPxEgx05duoLLKOaFne/75\nzCO5WItzShMRt7/i4DubW4FpRKc+P+XuC83sPiDi7lOAJ4HngskDm4kmEoJyE4hORKgAbnH3Svhq\n2vQ36gyO3waMBboChWY21d1vAqYSnUpdRHQ69Q3xarNIQ322bgcTZkUX59y0q4zOrbO5+YzejD4+\nn955rcIOT6RRWXSyl1RXUFDgkUgk7DCkidteGl2cc0KkhHnFW8lMN84b0IXRBfmc0TdPi3NKyjGz\n2e5eUF859dtFEqiqyvlkxSYmRkp4Y8EaSsurOKpLa35yyUBGHHMYHVtlhx2iSNwp8YgkwKqte5g8\nu4SJs4sp3ryH1jkZjDo+nysLejC4e1tNFJBmRYlHJE5Kyyt5a9E6JkSK+XvRRtzh1D4d+fG3tTin\nNG9KPCKNbMGqbUyMFPPyp6vZtqec7u1yue2cvow6Pp8eHbQ4p4gSj0gj2LKrjJc/XcWESAmL12wn\nK1icc/TxPfjWkVqcU6Q6JR6RBqqscj5ctoGJkRLeWrSOssoqhuS35ecjjuayIYfRtoUW5xSpjRKP\nyEH6YuMuJs0uYdLsEtZuL6V9i0yuOfkIRhfkM6CbFucUqY8Sj0gMdpdV8Mb8tUyIFDNjRXRxzjP7\n5XHPpQM5d4AW5xQ5GEo8InVwd+YWb2VipJhX561h594KenZswe0XHMXI4/Lp2laLc4o0hBKPSA3r\nd5Ty0pzo4pyfB4tzXjy4G1ee0IMCLc4pcsiUeESILs45fcl6JkRKmL50PZVVTsER7Xlo5JFcNKQb\nrbQ4p0ij0b8madaWrdvBxNklvDinhI07y8hrnc33Tu/N6IJ8jtTinCJxocQjzc6O0nJeK1zDhEgx\nc1duJSPNOHdAZ64s6MGZ/bQ4p0i8KfFIs1BV5cxYsZmJs4uZOj+6OGe/Lq24++IBjDi2O520OKdI\nwijxSJP39EcrePqjL1i5eTetszMYeVx0cc4h+VqcUyQMSjzSpE2dv4Z7X13ECT3b86Pz+3HBoK7k\nZmlxTpEwKfFIk7V5Vxk/fWUBg7u35fnvnazvbkSShBKPNFn3vbqQbXvK+b+bTlLSEUki+tcoTdLb\ni9bx8qerueXsPvTvqvXTRJKJEo80Odv2lPOfL8+nf9fW/MtZfcIOR0Rq0FCbNDm/fH0xG3eW8adr\nT9DinSJJSP8qpUn5cNkGXogUc/MZvRmc3zbscESkFko80mTs3FvBnZPn0zuvJf92bt+wwxGROmio\nTZqMh95cwupte5j0z6eQk6nf6ogkK/V4pEmYuWIzf/7Hl9zwrV4cf0SHsMMRkQNQ4pGUt6eskrGT\n5nF4hxb8+IJ+YYcjIvXQUJukvEfe/owvNu3mrzedRIss/UmLJDv1eCSlzV25hT99uJx/OulwvtWn\nU9jhiEgMlHgkZe2tqGTspEK6tMlh3IX9ww5HRGKkcQlJWb99t4hl63fy9A0n0DonM+xwRCRG6vFI\nSlqwahuPv/c5I4/L5+yjOocdjogcBCUeSTnllVWMnVRIh5ZZ/OSSAWGHIyIHSUNtknL+8P7nLFqz\nnT9893jatcgKOxwROUjq8UhKWbZuB//7ThGXDOnGBYO6hh2OiDRAXBOPmQ0zs6VmVmRmd9ZyPtvM\nXgjOzzCzntXOjQuOLzWzC+qr08x6BXUsC+rMCo4fbmbTzWyumRWa2UXxbLPET2WVc/ukQlrlZHDv\nZYPCDkdEGihuicfM0oHHgAuBgcB3zGxgjWI3AlvcvQ/wCPBgcO1AYAwwCBgGPG5m6fXU+SDwiLv3\nBbYEdQPcDUxw92ODOh+PR3sl/p7+aAWfFm/lnksH0rFVdtjhiEgDxbPHcyJQ5O7L3b0MGA8Mr1Fm\nOPBssD0JONfMLDg+3t33uvsKoCior9Y6g2vOCeogqHNEsO3AvltQtgVWN3I7JQFWbNzFw9OWct6A\nLlw29LCwwxGRQxDPxNMdKK62XxIcq7WMu1cA24COB7i2ruMdga1BHTWf62fANWZWAkwF/rW2YM3s\nZjOLmFlkw4YNsbdS4q6qyrljciFZGWn84vKjiX7OEJFUFc/EU9u7g8dYprGOA3wHeMbd84GLgOfM\nbL92u/sT7l7g7gV5eXm1VCdh+cuML5m5YjM/uWQgXdrkhB2OiByieCaeEqBHtf189h/m+qqMmWUQ\nHQrbfIBr6zq+EWgX1FHzuW4EJgC4+z+AHECLeqWI4s27eeCNJZzetxOjj88POxwRaQTxTDyzgL7B\nbLMsol/sT6lRZgpwXbA9CnjX3T04PiaY9dYL6AvMrKvO4JrpQR0Edb4SbK8EzgUwswFEE4/G0lKA\nu3PXS/Mx4IErBmuITaSJiNsPSN29wsxuBaYB6cBT7r7QzO4DIu4+BXiS6NBXEdGezpjg2oVmNgFY\nBFQAt7h7JUBtdQZPeQcw3szuB+YGdQP8B/BHM/t3osNv1weJSpLcxEgJHy7byM9HHE1++xZhhyMi\njcT0Hry/goICj0QiYYfRrK3dVsr5j7zPwG5teP57J5OWpt6OSLIzs9nuXlBfOa1cIEnH3bn75fmU\nV1bx4MghSjoiTYwSjySdKfNW8/bi9fz420fRs1PLsMMRkUYWU+Ixs1PNrGWwfY2Z/drMjohvaNIc\nbdy5l59NWcixh7fjhlN7hR2OiMRBrD2e3wG7zWwoMBb4Evhz3KKSZuueKQvZtbeSh0YOIV1DbCJN\nUqyJpyKYCTYceNTdHwVaxy8saY7eXLCG1wvX8G/n9aVvF/15iTRVsU6n3mFm44BrgDOCxTp1r2Fp\nNFt3l3H3ywsZdFgbbj6jd9jhiEgcxdrjuQrYC9zo7muJroP2cNyikmbnvtcWsXV3GQ+NGkJmuua8\niDRlMfd4iA6xVZpZP6A/8Hz8wpLmZPqS9bw4ZxW3ndOHQYe1DTscEYmzWD9afgBkm1l34B3gBuCZ\neAUlzcf20nLuemk+/bq04pZz+oQdjogkQKyJx9x9N3AF8Bt3v5zoTdpEDskDU5ewbnspD40aSnZG\netjhiEgCxJx4zOwU4Grg9eCY3iXkkHxUtJHnZ67ke6f35pge7cIOR0QSJNbE80NgHPBSsIBnb6Kr\nQYs0yO6yCu58sZBenVry7+f3CzscEUmgmCYXuPv7wPtm1trMWrn7cuC2+IYmTdnD05ZSsmUPE75/\nCjmZ6jyLNCexLpkz2MzmAguARWY228z0HY80SOSLzTzz8Rdce/IRnNCzQ9jhiEiCxTrU9gfgR+5+\nhLsfTnCPm/iFJU1VaXklYycVcljbXMYO6x92OCISglgTT0t3/+o7HXd/D9CywXLQ/uftZSzfuIsH\nRw6hZXbc7kMoIkks1n/5y83sJ8Bzwf41wIr4hCRN1bzirTzxweeMOaEHp/XtFHY4IhKSWHs8/w/I\nA14EXgq2b4hXUNL0lFVUMXZSIZ1b53DXxQPCDkdEQhTrrLYtaBabHILHphexdN0OnryugDY5Wl9W\npDk7YOIxs1cBr+u8u1/W6BFJk7N4zXYem17E5cd259wBXcIOR0RCVl+P578SEoU0WRWVVdw+aR7t\nWmTy00sGhh2OiCSBAyae4IejIg32xIfLWbBqO7+7+jjat8wKOxwRSQIxfcdjZvPZf8htGxAB7nf3\nTY0dmKS+ovU7+Z+3l3HR4K5cOLhb2OGISJKIdTr1G0Al8NdgfwxgRJPPM8CljR6ZpLTKKmfspHm0\nyErn3suODjscEUkisSaeU9391Gr7883sI3c/1cyuiUdgktqe/fgL5qzcyiNXDSWvdXbY4YhIEon1\ndzytzOykfTtmdiLQKtitaPSoJKV9uWkXD01bwjn9OzPimO5hhyMiSSbWHs9NwFNm1oroENt24EYz\nawk8EK/gJPVUVTl3Tp5PZloav7j8aMws7JBEJMnE+gPSWcBgM2tL9G6kW6udnhCXyCQlPT9rJf9Y\nvolfXTGYbm1zww5HRJJQrLdFaGtmvwbeAd42s/8OkpDIV1Zt3cMDU5dwap+OXHVCj7DDEZEkFet3\nPE8BO4Arg8d24Ol4BSWpx92568X5VLnzqyuGaIhNROoU63c8R7r7yGr795rZp/EISFLT5DmreP+z\nDdx72SB6dGgRdjgiksRi7fHsMbPT9u2Y2anAnviEJKlm/fZS7nt1ISf0bM93Tz4i7HBEJMnF2uP5\nAfDsvskFwGbg+ngFJanD3bn75QXsrajiwZFDSEvTEJuIHFiss9o+BYaaWZtgf3tco5KU8fr8Nfxt\n0TrGXdif3nmt6r9ARJq9+m6L8KM6jgPg7r+OQ0ySIjbt3Ms9ryxkaH5bbjytV9jhiEiKqO87ntb1\nPA7IzIaZ2VIzKzKzO2s5n21mLwTnZ5hZz2rnxgXHl5rZBfXVaWa9gjqWBXVmVTt3pZktMrOFZrZv\nvTk5RPe+uojtpeU8NGooGemxfl0oIs1dfbdFuLehFZtZOvAYcD5QAswysynuvqhasRuBLe7ex8zG\nAA8CV5nZQKILkQ4CDiP626F+wTV11fkg8Ii7jzez3wd1/87M+gLjiK43t8XMOje0TfK1vy1cy5R5\nq/nR+f04qmu9n0FERL5y0B9TzWxOjEVPBIrcfbm7lwHjgeE1ygwHng22JwHnWnQcbzgw3t33uvsK\noCior9Y6g2vOCeogqHNEsP094LHg9t24+/qDa7HUtG13OXe/vIAB3drwg7OODDscEUkxDRkfiXXa\nUneguNp+SXCs1jLuXkH0NgsdD3BtXcc7AluDOmo+Vz+gn5l9ZGafmNmwWhtldrOZRcwssmHDhhib\n2Dzd//oiNu0q4+FRQ8jUEJuIHKSGvGu8HmO52hJUzZvJ1VWmsY5DdDixL3AW8B3gT2bWbr/C7k+4\ne4G7F+Tl5dVSnQC8/9kGJs4u4Z/P7M3R3bVqkogcvINOPO5+d4xFS4DqC3blA6vrKmNmGUBbor8R\nquvauo5vBNoFddR8rhLgFXcvD4btlhJNRHKQdpSWM25yIX06t+Jfz9H/QhFpmFgXCd1hZttrPIrN\n7CUz613HZbOAvsFssyyikwWm1CgzBbgu2B4FvOvuHhwfE8x660U0Ucysq87gmulBHQR1vhJsvwyc\nHbSjE9Ght+WxtFu+6cE3l7BmeykPjRpCTmZ62OGISIqKdeWCXxPtQfyV6LDWGKAr0d7DU0SHsb7B\n3SvM7FZgGpAOPOXuC83sPiDi7lOAJ4HnzKyIaE9nTHDtQjObACwieqO5W9y9EqC2OoOnvAMYb2b3\nA3ODugnKftvMFhG9ffft7r4pxnZL4JPlm/i/T1Zy02m9OO7w9mGHIyIpzKKdhXoKmc1w95NqHPvE\n3U82s3nuPjRuEYagoKDAI5FI2GEkjT1llQx79AMA3vy3M8jNUm9HRPZnZrPdvaC+crF+x1MV/Agz\nLXhcWe1c/ZlLUtp//20pX27azYMjhyjpiMghizXxXA18F1gPrAu2rzGzXODWOMUmSWDOyi08+dEK\nrjn5cE7u3THscESkCYh1kdDlwKV1nP5744UjyaS0vJKxkwo5rG0ud144IOxwRKSJiHVWWz8ze8fM\nFgT7Q8ws1mnVkqJ+8+4yitbv5JdXDKZVdqzzUEREDizWobY/El3vrBzA3QsJZqBJ07Rg1TZ+//5y\nRh+fz5n99INaEWk8sSaeFu4+s8axilpLSsorq6jixxPn0bFlFndfPDDscESkiYl1/GSjmR1JMIPN\nzEYBa+IWlYTq9+9/zpK1O/jjtQW0bZEZdjgi0sTEmnhuAZ4A+pvZKmAF0Zlu0sQsXbuD37y7jMuG\nHsb5A7uEHY6INEGxJp5VwNNEl6XpAGwnuizNfXGKS0JQUVnF2EnzaJOTyc8uGxR2OCLSRMWaeF4B\ntgJz2H+hT2kinvpoBfNKtvHbfzqWDi2z6r9ARKQBYk08+e5e631spGlYvmEn//23z/j2wC5cPLhb\n2OGISBMW66y2j81scFwjkdBUVTl3TC4kOyON+0ccTfSGriIi8RFrj+c04HozWwHsJbpCtbv7kLhF\nJgnz3CdfMuuLLfzX6KF0bpMTdjgi0sTFmngujGsUEprizbt58M0lnNkvj5HH1bwzuYhI44t1rbYv\n4x2IJJ67c+eLhaSZ8csrBmuITUQS4qBvfS1NxwuzivmoaBPjLupP93a5YYcjIs2EEk8ztWbbHn7x\n+mJO6d2R75xweNjhiEgzosTTDLk7d704n4oq51cjB5OWpiE2EUkcJZ5m6OVPVzF96QZuv+AojujY\nMuxwRKSZUeJpZjbs2Mu9ry7i+CPac923eoYdjog0Q0o8zcw9Uxawu6ySB0cOIV1DbCISAiWeZmTq\n/DVMnb+WH57Xlz6dW4Udjog0U0o8zcSWXWX89JUFDO7elptP7x12OCLSjMW6coGkuPteW8TW3eU8\nd+NJZKTr84aIhEfvQM3AO4vX8dLcVdxydh8GdGsTdjgi0swp8TRx2/aUc9dL8zmqS2tuObtP2OGI\niGioral7YOpiNuzYyx+vLSArQ58zRCR8eidqwj5ctoHxs4q5+YwjGZLfLuxwREQAJZ4ma9feCu6c\nPJ/eeS354Xl9ww5HROQrGmproh6etpTV2/Yw8funkJOZHnY4IiJfUY+nCZq5YjPPfPwF13+rJwU9\nO4QdjojINyjxNDGl5ZXcMbmQHh1yuf2Co8IOR0RkPxpqa2IeeeszVmzcxV9uOokWWXp5RST5qMfT\nhHxavJU/fric75x4OKf26RR2OCIitYpr4jGzYWa21MyKzOzOWs5nm9kLwfkZZtaz2rlxwfGlZnZB\nfXWaWa+gjmVBnVk1nmuUmbmZFcSnteHaW1HJ2Enz6NImh3EX9Q87HBGROsUt8ZhZOvAYcCEwEPiO\nmQ2sUexGYIu79wEeAR4Mrh0IjAEGAcOAx80svZ46HwQecfe+wJag7n2xtAZuA2bEo63J4LF3i/hs\n3U5+eflg2uRkhh2OiEid4tnjOREocvfl7l4GjAeG1ygzHHg22J4EnGtmFhwf7+573X0FUBTUV2ud\nwTXnBHUQ1Dmi2vP8HHgIKG3sRiaDhau38fh7n3PFcd05u3/nsMMRETmgeCae7kBxtf2S4FitZdy9\nAtgGdDzAtXUd7whsDer4xnOZ2bFAD3d/7UDBmtnNZhYxs8iGDRtibWPoyiurGDupkHYtsvjpJTU7\nlCIiySeeiae221t6jGUa5biZpREdwvuPA8QZLez+hLsXuHtBXl5efcWTxhMfLGfh6u3cP+Jo2rXI\nqv8CEZGQxTPxlAA9qu3nA6vrKmNmGUBbYPMBrq3r+EagXVBH9eOtgaOB98zsC+BkYEpTmWBQtH4H\nj769jIuHdGPY0V3DDkdEJCbxTDyzgL7BbLMsopMFptQoMwW4LtgeBbzr7h4cHxPMeusF9AVm1lVn\ncM30oA6COl9x923u3snde7p7T+AT4DJ3j8Sr0YlSWeXcPqmQltnp3HvZoLDDERGJWdx+YejuFWZ2\nKzANSAeecveFZnYfEHH3KcCTwHNmVkS0pzMmuHahmU0AFgEVwC3uXglQW53BU94BjDez+4G5Qd1N\n1tMfrWDuyq08OuYYOrXKDjscEZGYWbSzINUVFBR4JJK8naIvNu5i2KMfcFqfTvzx2gKik/pERMJl\nZrPdvd6vMrRyQYqpqnLumFxIZnoa948YrKQjIilHiSfF/GXmSmas2MxPLh5I17Y5YYcjInLQlHhS\nSMmW3fxq6mJO79uJ0QX5YYcjItIgSjwpwt0Z9+J8AB64QkNsIpK6lHhSxMTZJXy4bCN3Xtif/PYt\nwg5HRKTBlHhSwLrtpfz8tUWc2KsDV590RNjhiIgcEiWeJOfu/OdLCyivrOKhkUNIS9MQm4ikNiWe\nJPdq4RreXryOH3/7KHp2ahmFIvelAAALGElEQVR2OCIih0yJJ4lt2rmXn01ZyDE92nHDqb3CDkdE\npFEo8SSxe6YsZGdpBQ+NGkK6hthEpIlQ4klSby5Yy2uFa7jt3D7069I67HBERBqNEk8S2rq7jJ+8\nsoCB3drw/TOPDDscEZFGFbfVqaXhfv7aYrbsKuOZG04gM12fDUSkadG7WpKZvnQ9k+eU8IOzjmTQ\nYW3DDkdEpNEp8SSRHaXl3PXifPp2bsWt5/QJOxwRkbhQ4kkiD7yxhHXbS3lo1BCyM9LDDkdEJC6U\neJLEx59v5K8zVnLT6b059vD2YYcjIhI3SjxJYHdZBXdOnk+vTi350fn9wg5HRCSuNKstCfzXtM9Y\nuXk3E75/CjmZGmITkaZNPZ6Qzf5yM09/vIJrTzmCE3t1CDscEZG4U+IJUWl5JbdPKuSwtrmMHdY/\n7HBERBJCQ20hevSdZSzfsIvnbjyRVtl6KUSkeVCPJySFJVt54oPlXFXQg9P75oUdjohIwijxhKCs\nooqxkwrp1CqLuy4eEHY4IiIJpfGdEDz+XhFL1u7gyesKaJubGXY4IiIJpR5Pgi1es53fvlvEiGMO\n49wBXcIOR0Qk4ZR4EqiiMjrE1q5FJvdcOijscEREQqGhtgT6099XMH/VNh6/+jjat8wKOxwRkVCo\nx5Mgn2/Yya/f+owLj+7KRYO7hR2OiEholHgSoLLKGTupkNzMdO4driE2EWnelHgS4M//+ILZX27h\nnksH0rl1TtjhiIiESoknzlZu2s1Dby7l7KPyuPzY7mGHIyISOiWeOHJ37phcSEaa8csrBmNmYYck\nIhI6JZ44en5mMf9Yvom7Lh5At7a5YYcjIpIU4pp4zGyYmS01syIzu7OW89lm9kJwfoaZ9ax2blxw\nfKmZXVBfnWbWK6hjWVBnVnD8R2a2yMwKzewdMzsinm3eZ/XWPfxy6mJO7dORMSf0SMRTioikhLgl\nHjNLBx4DLgQGAt8xs4E1it0IbHH3PsAjwIPBtQOBMcAgYBjwuJml11Png8Aj7t4X2BLUDTAXKHD3\nIcAk4KF4tLc6d+eul+ZTWeX86oohGmITEakmnj2eE4Eid1/u7mXAeGB4jTLDgWeD7UnAuRZ9lx4O\njHf3ve6+AigK6qu1zuCac4I6COocAeDu0919d3D8EyA/Dm39hhfnrOK9pRu4Y9hR9OjQIt5PJyKS\nUuKZeLoDxdX2S4JjtZZx9wpgG9DxANfWdbwjsDWoo67ngmgv6I3agjWzm80sYmaRDRs21Nu4uqzf\nUcp9ry2i4Ij2XHtKzwbXIyLSVMUz8dQ2vuQxlmms418/kdk1QAHwcC1lcfcn3L3A3Qvy8hp2fxx3\n5ycvL6C0vJKHRg0hLU1DbCIiNcUz8ZQA1b9VzwdW11XGzDKAtsDmA1xb1/GNQLugjv2ey8zOA/4T\nuMzd9x5Sqw5g6vy1TFu4jn8/vx+981rF62lERFJaPBPPLKBvMNssi+hkgSk1ykwBrgu2RwHvursH\nx8cEs956AX2BmXXVGVwzPaiDoM5XAMzsWOAPRJPO+ji1FYBWORl8e2AXbjqtVzyfRkQkpcVtdWp3\nrzCzW4FpQDrwlLsvNLP7gIi7TwGeBJ4zsyKiPZ0xwbULzWwCsAioAG5x90qA2uoMnvIOYLyZ3U90\nJtuTwfGHgVbAxGB22Up3vywebT6zXx5n9tNtrEVEDsSinQWprqCgwCORSNhhiIikFDOb7e4F9ZXT\nygUiIpJQSjwiIpJQSjwiIpJQSjwiIpJQSjwiIpJQSjwiIpJQSjwiIpJQ+h1PLcxsA/BlAy/vRHQJ\nn6ZAbUk+TaUdoLYkq0NpyxHuXu+v6JV4GpmZRWL5AVUqUFuST1NpB6gtySoRbdFQm4iIJJQSj4iI\nJJQST+N7IuwAGpHaknyaSjtAbUlWcW+LvuMREZGEUo9HREQSSolHREQSSomngcxsmJktNbMiM7uz\nlvPZZvZCcH6GmfVMfJSxiaEt15vZBjP7NHjcFEac9TGzp8xsvZktqOO8mdn/Bu0sNLPjEh1jrGJo\ny1lmtq3aa/LTRMcYCzPrYWbTzWyxmS00s3+rpUxKvC4xtiVVXpccM5tpZvOCttxbS5n4vYe5ux4H\n+SB699PPgd5AFjAPGFijzL8Avw+2xwAvhB33IbTleuC3YccaQ1vOAI4DFtRx/iLgDcCAk4EZYcd8\nCG05C3gt7DhjaEc34LhguzXwWS1/XynxusTYllR5XQxoFWxnAjOAk2uUidt7mHo8DXMiUOTuy929\nDBgPDK9RZjjwbLA9CTjXgntvJ5lY2pIS3P0DordQr8tw4M8e9QnQzsy6JSa6gxNDW1KCu69x9znB\n9g5gMdC9RrGUeF1ibEtKCP5f7wx2M4NHzZlmcXsPU+JpmO5AcbX9Evb/A/yqjLtXANuAjgmJ7uDE\n0haAkcEwyCQz65GY0BpdrG1NFacEQyVvmNmgsIOpTzBUcyzRT9fVpdzrcoC2QIq8LmaWbmafAuuB\nt9y9ztelsd/DlHgaprasX/PTQixlkkEscb4K9HT3IcDbfP0pKNWkymsSizlE18UaCvwGeDnkeA7I\nzFoBk4Efuvv2mqdruSRpX5d62pIyr4u7V7r7MUA+cKKZHV2jSNxeFyWehikBqn/qzwdW11XGzDKA\ntiTn0Em9bXH3Te6+N9j9I3B8gmJrbLG8binB3bfvGypx96lAppl1CjmsWplZJtE36r+4+4u1FEmZ\n16W+tqTS67KPu28F3gOG1TgVt/cwJZ6GmQX0NbNeZpZF9Iu3KTXKTAGuC7ZHAe968C1dkqm3LTXG\n2y8jOradiqYA1wazqE4Gtrn7mrCDaggz67pvvN3MTiT6b3lTuFHtL4jxSWCxu/+6jmIp8brE0pYU\nel3yzKxdsJ0LnAcsqVEsbu9hGY1RSXPj7hVmdiswjeissKfcfaGZ3QdE3H0K0T/Q58ysiOinhDHh\nRVy3GNtym5ldBlQQbcv1oQV8AGb2PNFZRZ3MrAS4h+iXprj774GpRGdQFQG7gRvCibR+MbRlFPAD\nM6sA9gBjkvSDzanAd4H5wfcJAHcBh0PKvS6xtCVVXpduwLNmlk40OU5w99cS9R6mJXNERCShNNQm\nIiIJpcQjIiIJpcQjIiIJpcQjIiIJpcQjIiIJpcQjEhIz21l/qQNeP8nMegfbrczsD2b2ebDa8Adm\ndpKZZQXb+umEJA0lHpEUFKwBlu7uy4NDfyL6W4u+7j6I6G+tOgULv74DXBVKoCK1UOIRCVnwi/2H\nzWyBmc03s6uC42lm9njQg3nNzKaa2ajgsquBV4JyRwInAXe7exVAsNr460HZl4PyIklB3W+R8F0B\nHAMMBToBs8zsA6K/lO8JDAY6E12q6KngmlOB54PtQcCn7l5ZR/0LgBPiErlIA6jHIxK+04Dng9WC\n1wHvE00UpwET3b3K3dcC06td0w3YEEvlQUIqM7PWjRy3SIMo8YiEr66bax3oplt7gJxgeyEw1MwO\n9O85GyhtQGwijU6JRyR8HwBXBTfmyiN62+uZwN+J3oAvzcy6EF00dJ/FQB8Ad/8ciAD3VlsZua+Z\nDQ+2OwIb3L08UQ0SORAlHpHwvQQUAvOAd4GxwdDaZKL3RFkA/IHo3S63Bde8zjcT0U1AV6DIzOYT\nvW/SvnvanE10BWiRpKDVqUWSmJm1cvedQa9lJnCqu68N7qEyPdiva1LBvjpeBMa5+9IEhCxSL81q\nE0lurwU37MoCfh70hHD3PWZ2D9AdWFnXxcHN/V5W0pFkoh6PiIgklL7jERGRhFLiERGRhFLiERGR\nhFLiERGRhFLiERGRhPr/lu98hZt1UGYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10d9a26d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# scores_：dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 9\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L1.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个score似乎和GridSearchCV得到的Score不一样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "惩罚不够，没有稀疏系数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.08350778, -0.08351508, -0.08351904, -0.08351952],\n",
       "        [-0.08843242, -0.08868946, -0.08872516, -0.08872888],\n",
       "        [-0.08309293, -0.08322307, -0.08324103, -0.0832429 ],\n",
       "        [-0.08219928, -0.08218897, -0.0821904 , -0.08219058],\n",
       "        [-0.08077456, -0.08076276, -0.0807651 , -0.08076537]]),\n",
       " 1: array([[-0.31683143, -0.31682081, -0.31682044, -0.31682046],\n",
       "        [-0.32034989, -0.32039856, -0.32040339, -0.32040501],\n",
       "        [-0.31524611, -0.31526643, -0.31527065, -0.31526969],\n",
       "        [-0.31669067, -0.31669   , -0.31669026, -0.31669034],\n",
       "        [-0.31924003, -0.31929319, -0.3192987 , -0.31930054]]),\n",
       " 2: array([[-0.26142898, -0.26141999, -0.26142016, -0.26142016],\n",
       "        [-0.26187226, -0.2618826 , -0.26188465, -0.26188486],\n",
       "        [-0.26763476, -0.26772384, -0.26773416, -0.26773516],\n",
       "        [-0.26382995, -0.26383176, -0.26383281, -0.2638332 ],\n",
       "        [-0.26330204, -0.26329121, -0.26329121, -0.26329116]]),\n",
       " 3: array([[-0.1306382 , -0.1308083 , -0.13083052, -0.13083283],\n",
       "        [-0.12744279, -0.12747566, -0.12748451, -0.12748547],\n",
       "        [-0.12538032, -0.12529983, -0.12529475, -0.12529424],\n",
       "        [-0.12722342, -0.12721798, -0.12722184, -0.12722229],\n",
       "        [-0.13054192, -0.13073497, -0.13076654, -0.13077007]]),\n",
       " 4: array([[-0.01576954, -0.01553215, -0.01582445, -0.01588949],\n",
       "        [-0.01307025, -0.01208326, -0.01231789, -0.01240916],\n",
       "        [-0.01500351, -0.01452187, -0.01465487, -0.01468109],\n",
       "        [-0.01831463, -0.01852061, -0.01933046, -0.01958359],\n",
       "        [-0.01547824, -0.01405319, -0.01392112, -0.01392137]]),\n",
       " 5: array([[-0.11886466, -0.11892057, -0.11892649, -0.11892708],\n",
       "        [-0.11651845, -0.11662144, -0.11663228, -0.11663337],\n",
       "        [-0.10609177, -0.10607252, -0.10607079, -0.10607062],\n",
       "        [-0.10414034, -0.10414877, -0.10414982, -0.10414976],\n",
       "        [-0.10741697, -0.10740109, -0.10739972, -0.10739959]]),\n",
       " 6: array([[-0.10056363, -0.10063665, -0.10064584, -0.1006468 ],\n",
       "        [-0.11305698, -0.11313366, -0.1131425 , -0.11314307],\n",
       "        [-0.10452125, -0.10452889, -0.10453028, -0.10453041],\n",
       "        [-0.10079896, -0.10081567, -0.10081814, -0.1008184 ],\n",
       "        [-0.1007295 , -0.10074818, -0.10075083, -0.10075118]]),\n",
       " 7: array([[-0.10180337, -0.10178982, -0.10178902, -0.10178903],\n",
       "        [-0.11017447, -0.11031259, -0.11032783, -0.11032937],\n",
       "        [-0.10007752, -0.10007918, -0.10007977, -0.10007985],\n",
       "        [-0.10344305, -0.1034777 , -0.1034819 , -0.10348235],\n",
       "        [-0.10525239, -0.1053091 , -0.10531573, -0.1053164 ]]),\n",
       " 8: array([[-0.09435939, -0.09452746, -0.09455037, -0.09455274],\n",
       "        [-0.08331736, -0.08332715, -0.08333101, -0.08333147],\n",
       "        [-0.09284679, -0.0928926 , -0.09289928, -0.09289997],\n",
       "        [-0.08997929, -0.09009711, -0.09011306, -0.09011489],\n",
       "        [-0.09095339, -0.0909491 , -0.09095116, -0.09095136]])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEKCAYAAAAiizNaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XeYVPXZ//H3zcIuZZe6FOkgSO/L\noqJGDRosiAq2IFI0Pibxl2qNLZqYqKhPEh9jX0qMmoiNWELsRlBhQVgWBOm9LX2B7ffvjzkkKy4w\nwM6cLZ/Xdc3lzJlzZj7HYffec5/vfI+5OyIiIvFSI+wAIiJSvajwiIhIXKnwiIhIXKnwiIhIXKnw\niIhIXKnwiIhIXKnwiIhIXKnwiIhIXKnwiIhIXNUMO0BFlJqa6u3btw87hohIpTJnzpwcd296pPVU\neMrQvn17MjMzw44hIlKpmNnqaNZTq01EROJKhUdEROJKhUdEROJKhUdEROJKhUdEROJKhUdEROJK\nhUdEROJKhUdERCgsLuGNeet54Ys1MX8vfYFURKQa27G3gBdmreEvn61m0+48+rVtyFXpbTCzmL2n\nCo+ISDW0dPMeJs5cxatz15FXWMJpnVL53aU9OfOkZjEtOqDCIyJSbZSUOJ8s3UrGjFV88vVWEmvW\n4NJ+rRg3uANdWqTELYcKj4hIFbevoIhX565n4oyVLN+6l2YpSdx07klcld6WJslJcc+jwiMiUkVt\n2LmfKZ+t5sVZa9i1v5BerRrwhyv6cn6vE0isGd7YMhUeEZEqZu6aHWR8upJ3sjfh7gzt2YLxgzsw\noF2jmJ+/iYYKj4hIFVBYXMI72ZvI+HQl89buJKV2TcYPbs81p7SnTeO6Ycf7BhUeEZFKbOe+yHDo\nKTMjw6HbN6nLvRf1YMSA1iQnVcxf8RUzlYiIHNayLXuYOGMVrwTDoQd3asL9l/TkrC7NqFEj/Hba\n4ajwiIhUEu7OJ0tzyPh0JR8Hw6Ev6duKcae1p2uL+mHHi5oKj4hIBbe/oJhXv1zHxBmrWLYll6Yp\nSfzynJP4/qBwhkMfr5gWHjMbCvwRSACedfcHDnr+DOAPQG/gSnefGixvB7wabFcLeMzdnwye+wg4\nAdgfvMy57r7FzG4AfgwUA7nA9e6+yMyaAFOBgcAkd78xhrssIlJuNu7673DonfsK6dmqPv97RR8u\n6NUy1OHQxytmhcfMEoDHgXOAdcBsM5vm7otKrbYGGAvcdNDmG4FT3T3fzJKB7GDbDcHzo9w986Bt\nXihVnC4CHgWGAnnAXUDP4CYiUqHNW7uTjE9X8vaCjZS4c273Fow/rQMD21eM4dDHK5ZHPOnAMndf\nAWBmLwHDgf8UHndfFTxXUnpDdy8o9TCJKGbRdvfdpR7WAzxYvhf41Mw6HdNeiIjEQVFxCf9cGBkO\nPXfNTlKSajL21PaMObXiDYc+XrEsPK2AtaUerwMGRbuxmbUB3gI6ATeXOtoBmGhmxcArwG/d3YNt\nfgz8AkgEzj6++CIisbdzXwEvzV7LlJmr2LArj3ZN6vLrYd0Zmdamwg6HPl6x3Kuyjgc92o3dfS3Q\n28xaAq+b2VR330ykzbbezFKIFJ7RwJRgm8eBx83s+8CdwJiow5pdD1wP0LZt22g3ExE5Jsu25DJp\n5kpembOe/YXFnHpiE+4b3pOzujYjoYIPhz5esSw864A2pR63BjYcYt1DcvcNZrYQOB2Y6u7rg+V7\nzOwFIi29KQdt9hLwxFG+z9PA0wBpaWlRF0gRkWi5O/9emkPGjJV8tCQyHPrivi0ZN7gD3U6oPMOh\nj1csC89soLOZdQDWA1cC349mQzNrDWxz9/1m1ggYDDxqZjWBhu6eY2a1gAuB94JtOrv70uAlLgCW\nlvXaIiLxtr+gmNe+jMwOvXRLLqnJSfx8yEmMOrktqZVwOPTxilnhcfciM7sRmE5kWHSGuy80s/uA\nTHefZmYDgdeARsAwM7vX3XsA3YBHzMyJtOwedvcFZlYPmB4UnQQiReeZ4C1vNLMhQCGwg1JtNjNb\nBdQHEs3sYiJDsEuPrhMRKXebduXxl89X8cIXa9ixr5AeLevzyGV9uLDPCSTVTAg7XmgsOC8vpaSl\npXlm5sGjtUVEojN/7U4yZqzkrayNFLtzbvfmjB/cgfQOjavEcOhDMbM57p52pPWq5pAJEZE4Kyou\nYfrCzWTMWMmc1TtITqrJmFPbM+aU9rRtUrWGQx8vFR4RkeOwa18hL81ew+RgOHTbxnW5Z1h3Rg5o\nTUrtWmHHq5BUeEREjsHyrblMmrGKqXPWsb+wmFM6NuHe4T05uxoMhz5eKjwiIlFyd2Ys20bGjJV8\nsHgLiQk1uKhvS8YNbk+Plg3CjldpqPCIiBxBXmExr3+5nowZK/l6cy6pyYn8bEhnRg1qR9OU6jcc\n+nip8IiIHMLm3Xn85bPV/PWL1ezYV0j3E+rz8GV9GFbNh0MfLxUeEZGDZK2LzA79ZjAc+pxuzRl/\nWgcGVfHh0PGiwiMiQmQ49L8WbSbj05VkBsOhrzmlPWNP1XDo8qbCIyLV2q79hfxt9homz1zN+p37\nadO4Dndd2J3L0zQcOlZUeESkWlqZs5dJM1by8px17CsoZlCHxtw9rDtDujXXcOgYU+ERkWrD3Zm5\nfBsZn67kgyVbqFWjBsP6RIZD92yl4dDxosIjIlVeXmExb8xbT8anq1iyeQ9N6iXyk7M7M+rktjRL\nqR12vGpHhUdEqqwtu/P4y+er+esXa9i+t4CuLVKYMLI3w/q0pHYtDYcOiwqPiFQ5C9btImPGSt7M\n2kBRiTOkW2R26JM7ajh0RaDCIyJVQnGJ8+6iTWR8uopZq7ZTLzGBUYPaMW5we9o1qRd2PClFhUdE\nKrXdeYX8ffZaJs1cxbod+2ndqA53XtCNywe2ob6GQ1dIKjwiUimtytnLpJmreDlzLXsLiknv0Jg7\nL+jOOd01HLqiU+ERkUrD3flseWR26PcXb6FmDWNYn5aMH9xBw6ErERUeEanw8gqLmTZvAxkzVrJ4\nU2Q49P87uzNXazh0paTCIyIV1pbdeTwfDIfeFgyHfmhEby7qq+HQlZkKj4hUONnrI8Oh/zE/Mhz6\nu12bMX5wB045sYmGQ1cBKjwiUiFEhkNvJmPGSmat3E7dYDj0mFPb0yFVw6GrEhUeEQnVwcOhWzWM\nDIe+LK0NDepoOHRVpMIjIqFYvW0vE2eUGg7dvjF3XtCNId2aUzOhRtjxJIZUeEQk7uas3s6VT38O\nwLDeLRk3uAO9Wms4dHWhwiMicZVXWMzNL2fRLKU2r/zwVFo00HDo6kaFR0Ti6tF3v2ZFzl7+et0g\nFZ1qSo1UEYmbuWt28Oy/V3BVelsGd0oNO46ERIVHROIi0mKbT4v6tfnV+V3DjiMhUqtNROLij+8v\nZfnWvUwen06KZo2u1nTEIyIxN3/tTp76eDlXpLXhOyc1DTuOhEyFR0RiKr+omJunzqdZSm3uuLBb\n2HGkAlCrTURi6rH3l/H15lwmjh2oC7MJoCMeEYmh7PW7eOLj5Yzo35qzujYLO45UEDEtPGY21MyW\nmNkyM7utjOfPMLO5ZlZkZiNLLW9nZnPMbJ6ZLTSzG0o991HwmvOCW7Ng+Q1mtiBY9qmZdS+1ze1B\nhiVm9r1Y7rOIRBQUlXDTy/NpUi+Ruy/sfuQNpNqIWavNzBKAx4FzgHXAbDOb5u6LSq22BhgL3HTQ\n5huBU90938ySgexg2w3B86PcPfOgbV5w9yeD974IeBQYGhSgK4EeQEvgPTM7yd2Ly21nReRbHv9w\nGYs37eHZa9JoUFctNvmvWB7xpAPL3H2FuxcALwHDS6/g7qvcPQsoOWh5gbvnBw+Tosnp7rtLPawH\neHB/OPCSu+e7+0pgWZBNRGJk0YbdPP7hMi7u25Ih3ZuHHUcqmFgWnlbA2lKP1wXLomJmbcwsK3iN\nB0sd7QBMDFpqd1mpq0KZ2Y/NbDnwEPCT8sghIkensDjSYmtYN5F7hvUIO45UQLEsPGVdJtDLWFYm\nd1/r7r2BTsAYMzvwZ9Mod+8FnB7cRpfa5nF3PxG4FbjzaHKY2fVmlmlmmVu3bo02pogc5MmPlrNo\n425+e3FPGtVLDDuOVECxLDzrgDalHrcGNhxi3UMKjnQWEikyuPv64L97gBcou232EnDx0eRw96fd\nPc3d05o21RfcRI7F4k27+dMHSxnWpyVDe7YIO45UULEsPLOBzmbWwcwSiZzgnxbNhmbW2szqBPcb\nAYOBJWZW08xSg+W1gAuB7OBx51IvcQGwNLg/DbjSzJLMrAPQGZh13HsnIt9QVFzCzS9nUb92Le69\nSC02ObSYjWpz9yIzuxGYDiQAGe6+0MzuAzLdfZqZDQReAxoBw8zsXnfvAXQDHjEzJ9Iqe9jdF5hZ\nPWB6UHQSgPeAZ4K3vNHMhgCFwA5gTJBjoZn9HVgEFAE/1og2kfL31CcrWLB+F38e1Z/GarHJYZh7\n1Kddqo20tDTPzDx4tLaIHMrSzXu44E+fMqR7M/48akDYcSQkZjbH3dOOtJ5mLhCR41JUXMJNU7Oo\nl5TAfcN7hh1HKgHN1SYix+W5T1cyf+1O/nRVP1KTk8KOI5WAjnhE5Jgt25LLI+9+zfd6NGdY7xPC\njiOVhAqPiByT4hLnlqnzqZuYwG8u7kmp73KLHJZabSJyTCbOWMncNTv5wxV9aZZSO+w4UonoiEdE\njtrKnL1MmL6EId2aMbxvy7DjSCWjwiMiR6UkaLEl1azB/Zf0UotNjpoKj4gclcmfrWL2qh3cPawH\nzeurxSZHT4VHRKK2etteHvrnEs7s0pQR/TXJuxwbFR4RiUqkxZZFzRrG7y9Vi02OXVSFx8wGB/Ok\nYWZXm9mjZtYuttFEpCL56xer+WLldu68sBsnNKgTdhypxKI94nkC2GdmfYBbgNXAlJilEpEKZe32\nffz+ncWc3jmVy9PaHHkDkcOItvAUeWQ20eHAH939j0BK7GKJSEXh7tz6ShY1zHhgRG+12OS4RfsF\n0j1mdjtwNXCGmSUAtWIXS0QqihdmrWHm8m387pJetGqoFpscv2iPeK4A8oFr3X0T0AqYELNUIlIh\nrNuxj9+99RWDOzXhqnS12KR8RH3EQ6TFVmxmJwFdgRdjF0tEwubu3P7qAhx44FK12KT8RHvE8wmQ\nZGatgPeBccCkWIUSkfD9PXMt/16aw+3ndaVN47phx5EqJNrCY+6+D7gUeMzdLwF0UXWRKmrjrv38\n9s2vOLljY0YN0jcnpHxFXXjM7BRgFPBWsCwhNpFEJEwHWmxFJc5DI/pQo4ZabFK+oi08PwNuB15z\n94Vm1hH4MHaxRCQsU+es46MlW7l1aBfaNlGLTcpfVIML3P1j4GMzSzGzZHdfAfwkttFEJN427crj\nvjcXkd6+Mdec0j7sOFJFRTtlTi8z+xLIBhaZ2Rwz0zkekSrE3bnjtQUUFJXw4MjearFJzETbansK\n+IW7t3P3tsAvgWdiF6ty2l9QzM9e+pKvN+8JO4rIUXt93nreX7yFm7/XhQ6p9cKOI1VYtIWnnrv/\n55yOu38E6F/mQbbvK2Dm8m2MyZjFhp37w44jErUte/L49bRF9G/bkHGDO4QdR6q4aAvPCjO7y8za\nB7c7gZWxDFYZtWpYh8nj08nNK2L0c1+wY29B2JFEjsjdufO1bPYXFjPhsj4kqMUmMRZt4RkPNAVe\nBV4L7o+LVajKrNsJ9Xl2TBprd+xn3KTZ7CsoCjuSyGH9I2sj/1q0mV+ecxInNk0OO45UA1EVHnff\n4e4/cff+7t7P3X/q7jtiHa6yGtSxCY9d1Y+sdTv58V/nUlhcEnYkkTJt3ZPPPW9k07dNQ647vWPY\ncaSaOOxwajP7B+CHet7dLyr3RFXE93q04P5LenH7qwu49ZUsHh6pL+JJxXPPtGz25hczYWRvtdgk\nbo70PZ6H45KiiroqvS05e/J55N2vaZqcxO3ndws7ksh/vJW1kbcXbOKWoV3o3FyX15L4OWzhCb44\nKsfhxrM7sTU3n6c+WUFqchI/OEPtDAnfttx87n4jm96tG3C9WmwSZ1HNXGBmC/h2y20XkAn81t23\nlXewqsLMuGdYD7blFnD/21/RJDmRS/u3DjuWVHP3TFvI7rxCXhh5MjUToh1jJFI+or0ezztAMfBC\n8PhKwIgUn0nAsHJPVoUk1DAevaIPO/YVcMvULBrVS+SsLs3CjiXV1D+zN/Jm1kZ+ec5JdGmhFpvE\nX7R/6gx299vdfUFwuwP4jrs/CLSPXbyqI6lmAk+NHkCXFin86Pm5fLlGgwIl/nbsLeDO17Pp0bI+\nN5x5YthxpJqKtvAkm9mgAw/MLB04MOBfX1SJUkrtWkwal06z+kmMmzSbZVs0tY7E173/WMjOfYVM\nGNmHWmqxSUii/Zd3HfCsma00s1XAs8B1ZlYP+P2hNjKzoWa2xMyWmdltZTx/hpnNNbMiMxtZanm7\nYCLSeWa20MxuKPXcR8FrzgtuzYLlvzCzRWaWZWbvm1m7Uts8aGbZwe2KKPc5JpqmJDFlfDo1a9Tg\nmudmsXGXptaR+Hh30WZen7eBG8/uRPeW9cOOI9VYtF8gne3uvYC+QF937x0s2+vufy9rGzNLAB4H\nzgO6A1eZWfeDVlsDjOW/544O2Aic6u59gUHAbWbWstTzo9y9b3DbEiz7Ekhz997AVOChIMcFQP8g\n+yDgZjML9aeuXZN6TBo3kN15RYzJmMXOfZpaR2Jr574CfvXaArq2SOFHZ3YKO45Uc9FeFqGBmT0K\nvA+8Z2aPmFmDI2yWDixz9xXuXgC8BAwvvYK7r3L3LKDkoOUF7p4fPEyKJqe7fxhcnhvgc+DA0LHu\nwMfuXuTue4H5wNAjvV6s9WzVgKevGcCqnH1cNzmT/QXFYUeSKuy+NxexfW8BD1/Wh8SaarFJuKL9\nF5gB7AEuD267gYlH2KYVsLbU43XBsqiYWRszywpe40F331Dq6YlBm+0uMyvr69bXEhmJB5FCc56Z\n1TWzVOAsoE20OWLp1BNT+cOVfZmzZgc3vjCXIk2tIzHwweLNvDp3PT8680R6tjrS34sisRdt4TnR\n3e8Jjl5WuPu9wJG+dVZWQTjk9DvfWtF9bdA26wSMMbPmwVOjgrbf6cFt9Dfe1OxqIA2YELzOv4C3\ngZnAi8BnlDEgwsyuN7NMM8vcunVrtDGP2/m9TuC+4T15f/EWbn91Ae5R/y8SOaJd+wv51avZnNQ8\nmRvPVotNKoZoC89+MzvtwAMzGwwc6az4Or55ZNEa2HCIdQ8pONJZSKTI4O7rg//uIXJuKL1UriHA\nHcBFpVp1uPv9wfmgc4gUxKVlvM/T7p7m7mlNmzY92pjHZfTJ7fjpdzvz8px1TJi+JK7vLVXb/W8t\nYmtuPg9f1oekmglhxxEBov8C6Q+BycF5HQO2ExkUcDizgc5m1gFYT+RLp9+P5s3MrDWwzd33m1kj\nYDDwqJnVBBq6e46Z1QIuBN4LtulH5EqpQ0sNODgwyKGhu28zs95Ab+BfUe533PxsSGe25ubz54+W\nk5qcxPjTdDEuOT4ff72Vv2eu44dnnkjv1g3DjiPyH1EVHnefB/Q5MBrM3XdHsU2Rmd0ITAcSgAx3\nX2hm9wGZ7j7NzAYSub5PI2CYmd3r7j2AbsAjZuZECt3D7r4gGL49PSg6CUSKzoFLcE8g8t2il4PT\nPmuC2bNrAf8Olu0Grnb3CvfdIzPjN8N7sj23gPveXEST5ESG9436lJjIN+zJK+S2V7Lo1CyZn363\nc9hxRL7BDndOwcx+cbiN3f3Rck9UAaSlpXlmZmYo751XWMyYjFnMXbOD58YM5IyT4tv2k6rh9lcX\n8LfZa3jlh6fSr22jsONINWFmc9w97UjrHekcT8oRblLOatdK4JkxaXRqlsINz89h/tqdYUeSSubT\npTm8OGsNPzi9o4qOVEiHPeKprsI84jlgy+48Rjw5k735xUy94RQ66pLEEoXc/CK+97+fkFSzBm//\n9HRq19KAAomf8jriKeuF5x5bJDkazerXZsr4QRgw+rlZbN6dF3YkqQQeeOcrNuzaz4TLeqvoSIV1\nLF9h1vVx46RDaj0mjUtn574CxmTMYtf+wrAjSQU2c3kOz3++hvGDOzCgXeOw44gc0rEUnrfKPYUc\nUq/WDXhqdBrLt+byg8mZ5BVqah35tr35Rdz6Shbtm9TlpnO7hB1H5LCOuvC4+52xCCKHdlrnVB69\nvC+zV2/nJy9+qal15FsmTF/Cuh37eWhkH+okqsUmFVu0k4TuMbPdB93WmtlrZqYLtsfBsD4tuefC\n7vxr0WbueiNbU+vIf3yxYhuTZq5izCntSe+gFptUfNHOXPAokeluXiByjudKoAWwhMgEomfGIpx8\n09jBHcjJLeD/PlxG0+QkfqGWSrW3v6CYW17Jom3jutwyVP8epHKItvAMdfdBpR4/bWafu/t9Zvar\nWASTsv3y3JPYuiefP32wjCbJSYw5tX3YkSREE6YvYfW2fbz4g5Opmxjtj7NIuKI9x1NiZpebWY3g\ndnmp59TziSMz4/5LenJO9+b8+h8LeTPrqOddlSoic9V2Js5cyeiT23HKiU3CjiMStWgLzygilx/Y\nAmwO7l9tZnWAG2OUTQ6hZkINHruqH2ntGvHzv81jxrKcsCNJnOUVFnPL1CxaNqjDbed1DTuOyFGJ\n9tLXK9x9mLununvT4P4yd9/v7p/GOqR8W+1aCTx7zUA6piZz/ZRMstfvCjuSxNGj737Nipy9PDSy\nN/WS1GKTyiXaUW0nmdn7ZpYdPO5tZhpWHbIGdWsxeXw6DesmMnbiLFbl7A07ksTB3DU7ePbfK7gq\nvS2DO6WGHUfkqEXbansGuB0oBHD3LCIj2yRkLRrUZsq16RSXONdkzGLLHk2tU5XlFRZz88vzaVG/\nNr86Xy02qZyiLTx13X3WQcsq3DVtqqsTmyYzcVw6W/fkMyZjNrvzNLVOVfXH95eyfOtefj+iNym1\na4UdR+SYRFt4cszsRIIRbGY2EtgYs1Ry1Pq2aciTowewdPMerp+iqXWqovlrd/LUx8u5Iq0N39F1\nmqQSi7bw/JjIZaW7mtl64GfADTFLJcfkOyc15eHL+vD5iu38/G/zKC7RSPeqIr+omJunzqdZSm3u\nuLBb2HFEjku0hWc9MBG4H3gJeBcYE6tQcuwu7teKOy/oxjvZm7hbU+tUGY+9v4yvN+fy+0t7UV8t\nNqnkoh2H+QawE5hLZOocqcCuO70jObkFPPnxcpql1OanQzqHHUmOQ/b6XTzx8XJG9G/NWV2bhR1H\n5LhFW3hau/vQmCaRcnXr0C5s3ZPP/773NU2SE7n65HZhR5JjUFBUwk0vz6dJvUTuvrB72HFEykW0\nhWemmfVy9wUxTSPlxsx4YEQvduwr4K43smlSL5Hzep0Qdiw5So9/uIzFm/bw7DVpNKirFptUDdGe\n4zkNmGNmS8wsy8wWmFlWLIPJ8auVUIPHv9+ffm0a8tOX5vHZ8m1hR5KjsHDDLh7/cBmX9GvFkO7N\nw44jUm6iLTznAZ2Bc4FhwIXBf6WCq5OYQMbYgbRtUpfrp2SycIOm1qkMCotLuPnlLBrWTeSeYWqx\nSdUS7Vxtq8u6xTqclI+GdROZMj6d5No1GTtxNmu27Qs7khzBEx8tZ9HG3fz24p40rJsYdhyRcnXU\nl76Wyqllwzr85dp0CotLuCbjC3Jy88OOJIeweNNuHvtgKcP6tGRozxZhxxEpdyo81UinZik8N2Yg\nm3bnMXbiLHLzNetRRVMUtNjq167FvRf1CDuOSEyo8FQzA9o14s+j+vPVxj38z18yyS/S1DoVyVOf\nrGDB+l385uKeNK6nFptUTSo81dDZXZvz0IjezFi2jV/+fT4lmlqnQli6eQ9/fG8p5/dqwfka+i5V\nmK4gVU2NGNCanNx8fv/OYprUS+TXF/XAzMKOVW0VFZdw09Qs6iUlcN/wnmHHEYkpFZ5q7H++cyI5\nufk88++VNE1J4sazNbVOWJ77dCXz1+7kT1f1IzU5Kew4IjGlwlPN3X5eN3JyC3j4X1+TmpzElelt\nw45U7Szbkssj737N93o0Z1hvtdik6lPhqeZq1DAeGtmb7XsL+NVrC2hcL5Fze2gIb7wUlzi3TJ1P\n3cQEfnNxT7U7pVrQ4AKhVkINnri6P71aN+T/vfgls1ZuDztStTFxxkrmrtnJr4f1oFlK7bDjiMSF\nCo8AUDexJhPHDqRVozpcO3k2izftDjtSlbcyZy8Tpi9hSLdmDO/bMuw4InET08JjZkODiUWXmdlt\nZTx/hpnNNbOi4HLaB5a3M7M5ZjbPzBaa2Q2lnvsoeM15wa1ZsPwXZrYomMT0fTNrV2qbh4LX+crM\n/mTqZ5Spcb3I1Dr1EmtyzXOzWLtdU+vESknQYkuqWYP7L+mlFptUKzErPGaWADxOZILR7sBVZnbw\nbIdrgLHACwct3wic6u59gUHAbWZW+k/CUe7eN7htCZZ9CaS5e29gKvBQkONUYDDQG+gJDAS+Uz57\nWfW0blSXyePTySssZkzGLLZpap2YmPzZKmav2sHdw3rQvL5abFK9xPKIJx1Y5u4r3L2AyCWzh5de\nwd1XuXsWUHLQ8gJ3P/AbLymanO7+obsf+BP9c6D1gaeA2kBi8Fq1gM3HtkvVQ5cWKTw3diDrd+5n\n/KTZ7NXUOuVq9ba9PPjPxZzVpSkj+rcKO45I3MWy8LQC1pZ6vC5YFhUzaxNc82ct8KC7l77k9sSg\nzXbXIdpm1wLvALj7Z8CHRI6iNgLT3f2ro9uV6mdg+8b83/f7k71hNzc8P4eCopIjbyRHFGmxZVGr\nRg1+d6labFI9xbLwlPUTFfXcLO6+NmibdQLGmNmBK2GNcvdewOnBbfQ33tTsaiANmBA87gR0I3IE\n1Ao428zO+FZYs+vNLNPMMrdu3RptzCrtnO7N+f0lvfj30hxunqqpdcrD81+s5ouV27nzwm6c0KBO\n2HFEQhHLwrMOaFPqcWtgwyHWPaTgSGchkSKDu68P/ruHyLmh9APrmtkQ4A7golKtukuAz909191z\niRwJnVzG+zzt7mnunta0adOjjVllXT6wDTd/rwtvzNvAb9/6CncVn2O1dvs+HnhnMad3TuXytDZH\n3kCkiopl4ZkNdDazDmaWCFwpf55ZAAARz0lEQVQJTItmQzNrbWZ1gvuNiAwOWGJmNc0sNVhei8iV\nULODx/2Ap4gUnS2lXm4N8J1g21pEBhao1XYUfnTmiYwb3J6MGSt58uMVYceplNydW1/JooYZD4zo\nrRabVGsxm7nA3YvM7EZgOpAAZLj7QjO7D8h092lmNhB4DWgEDDOze929B5HW2CNm5kRadg+7+wIz\nqwdMDwpIAvAe8EzwlhOAZODl4Id6jbtfRGSE29nAAiKtvn+6+z9itd9VkZlx1wXd2ZZbwIP/XEyT\n5ET9xX6UXpi1hpnLt/G7S3rRqqFabFK9mVon35aWluaZmZlhx6hwCopKuHbybGYu38bTowfw3W7N\nj7yRsG7HPr73v5/Qt21Dnr92kI52pMoysznunnak9TRzgUQtsWYNnrh6AD1a1ufHL8xlzmpNrXMk\n7s7try7AgQcuVYtNBFR45CglJ9UkY+xATmhQh/GTMvl6856wI1Vof89cy7+X5nD7eV1p07hu2HFE\nKgQVHjlqqclJTBmfTmLNGlzz3CzW79wfdqQKaeOu/fz2za84uWNjRg1qd+QNRKoJFR45Jm0a12XK\n+HT25hdxzXNfsGNvQdiRKpQDLbaiEuehEX2oUUMtNpEDVHjkmHU7oT7PjElj7Y79jJs0m30Fmlrn\ngKlz1vHRkq3cOrQLbZuoxSZSmgqPHJeTOzbhT1f2I2vdTn7017kUFmtqnU278rjvzUWkt2/MNae0\nDzuOSIWjwiPHbWjPFvz24l6Rv/CnZlXrqXXcnTteW0BhcQkPjuytFptIGXTpaykX3x/UlpzcfB59\n92uapiRx+/ndwo4Uitfnref9xVu484JudEitF3YckQpJhUfKzf87uxM5ufk89ckKUpOT+MEZHcOO\nFFdbdufx62mLGNCuEeMGdwg7jkiFpcIj5cbMuGdYD7blFnD/21/RJDmRS/u3PvKGVYC7c8fr2eQV\nFvPQyN4kqMUmckgqPFKuEmoYj17Rhx37CrhlahaN6iVyVpdmYceKuWnzN/Duos3cfl5XTmyaHHYc\nkQpNgwuk3CXVTOCp0QPo0iKFHz0/l7lrdoQdKaa27snn19MW0rdNQ647vXq1F0WOhQqPxERK7VpM\nGpdOs/pJjJ80m2Vbqu7UOne/kc3e/GImqMUmEhUVHomZpimRqXVq1ohMrbNxV9WbWuetrI28k72J\nn53Tmc7NU8KOI1IpqPBITLVrUo9J4wayO6+Ia56bxc59VWdqnW25+dz1Rja9WzfgerXYRKKmwiMx\n17NVA54ePYDV2/Zx3eRM9hcUhx2pXNwzbSF78gqZMLIPNRP0oyQSLf20SFyc2imVP1zZlzlrdnDj\nC3MpquRT6/wzeyNvZm3kJ2d3pksLtdhEjoYKj8TN+b1O4L7hPXl/8ZbIxdEq6dVvd+wt4M7Xs+nR\nsj43nHli2HFEKh19j0fiavTJ7di6J58/vb+U1JQkbh3aNexIR+3efyxk575CpowfRC212ESOmgqP\nxN3Ph3QmJzefJz5aTmpyEteeVnmml3l30WZen7eBnw3pTPeW9cOOI1IpqfBI3JkZvxnek+25Bfzm\nzUWkJicyvG+rsGMd0c59BfzqtQV0bZHCj87sFHYckUpLfQIJRUIN4w9X9mVQh8bc9PJ8Pvl6a9iR\njui+NxexfW8BD1/Wh8Sa+tEROVb66ZHQ1K6VwDNj0ujULIUbnp/D/LU7w450SB8s3syrc9fzozNP\npGerBmHHEanUVHgkVPVr12LyuIE0rpfIuEmzWbE1N+xI37JrfyG3v7qALs1TuPFstdhEjpcKj4Su\nWf3a/OXaQRgw+rlZbN6dF3akb7j/rUXk5BYw4bLeJNVMCDuOSKWnwiMVQofUekwal87OfQWMyZjF\nrv2FYUcC4KMlW/h75jquP6MjvVs3DDuOSJWgwiMVRq/WDXhy9ACWb83lB5MzySsMd2qd3XmRFlun\nZsn89LudQ80iUpWo8EiFcnrnpjxyeV9mr97OT178MtSpdX7/9lds3p3HhJG9qV1LLTaR8qLCIxXO\nRX1acs+F3fnXos3c9UZ2KFPrfLo0hxdnreUHp3ekX9tGcX9/kapMXyCVCmns4A5szc3n8Q8jsxv8\n8twucXvv3Pwibn0li46p9fj5OSfF7X1FqgsVHqmwbjq3Czl7Cnjsg2WkJicx5tT2cXnfB975ig27\n9jP1hlPUYhOJARUeqbDMjPsv6cm2vQX8+h8LaZKcyIW9W8b0PWcuy+H5z9dw7WkdGNCucUzfS6S6\n0jkeqdBqJtTg/77fj7R2jfj53+YxY1lOzN5rb34Rt76aRfsmdbkpjq09kepGhUcqvNq1Enj2moF0\nTE3m+imZZK/fFZP3mTB9Cet27OehkX2ok6gWm0isxLTwmNlQM1tiZsvM7LYynj/DzOaaWZGZjSy1\nvJ2ZzTGzeWa20MxuKPXcR8FrzgtuzYLlvzCzRWaWZWbvm1m7YPlZpdadZ2Z5ZnZxLPdbyl+DurWY\nPD6dhnUTGTtxFqty9pbr63+xYhuTZq5izCntSe+gFptILMWs8JhZAvA4cB7QHbjKzLoftNoaYCzw\nwkHLNwKnuntfYBBwm5mVbu6Pcve+wW1LsOxLIM3dewNTgYcA3P3DA+sCZwP7gH+V135K/LRoUJvJ\n49MpLnFGZ3zBlj3lM7XO/oJibnkli7aN63LLULXYRGItlkc86cAyd1/h7gXAS8Dw0iu4+yp3zwJK\nDlpe4O75wcOkaHIGBWZf8PBzoHUZq40E3im1nlQynZolM3FcOjl7ChiTMZvdecc/tc6E6UtYvW0f\nD47oTd1EjbcRibVYFp5WwNpSj9cFy6JiZm3MLCt4jQfdfUOppycGbbO7zMzK2Pxa4J0yll8JvBht\nBqmY+rZpyBNX92fp5j1cP+X4ptbJXLWdiTNXMvrkdpxyYpNyTCkihxLLwlNWQYj6K+juvjZom3UC\nxphZ8+CpUe7eCzg9uI3+xpuaXQ2kARMOWn4C0AuYXmZYs+vNLNPMMrdurfgXJavuzuzSjIcv68Pn\nK7bz87/No7jk6Gc3yCss5papWbRqWIfbzusag5QiUpZYFp51QJtSj1sDGw6x7iEFRzoLiRQZ3H19\n8N89RM4NpR9Y18yGAHcAF5Vq1R1wOfCau5fZm3H3p909zd3TmjZterQxJQQX92vFnRd0453sTdx9\nDFPrPPru16zI2cuDI3pTL0ktNpF4iWXhmQ10NrMOZpZIpM01LZoNzay1mdUJ7jcCBgNLzKymmaUG\ny2sBFwLZweN+wFNEis6WMl72KtRmq3KuO70j//Odjvz1izX88f2lUW83d80Onv33Cq5Kb8vgTqkx\nTCgiB4vZn3nuXmRmNxJpbSUAGe6+0MzuAzLdfZqZDQReAxoBw8zsXnfvAXQDHjEzJ9Kye9jdF5hZ\nPWB6UHQSgPeAZ4K3nAAkAy8Hp33WuPtFAGbWnsjR18ex2l8Jz21Du5Kzp4A/vLeU1OQkrj653WHX\nzyss5uaX59Oifm1+db5abCLxFtP+gru/Dbx90LK7S92fTRmjz9z9XaB3Gcv3AgMO8V5DDpNjFUcx\nsEEqFzPjgRG92LGvgLveyKZJvUTO63XCIdf/w3tLWb51L5PHp5NSu1Yck4oIaOYCqSJqJdTg8e/3\np1+bhvz0pXl8tnxbmevNX7uTpz9ZzhVpbfjOSTqXJxIGFR6pMuokJpAxdiBtm9Tl+imZLNzwzal1\n8ouKuenl+TRLqc0dF3YLKaWIqPBIldKwbiJTxqeTXLsmYzJms2bbf78r/Nj7y1i6JZffX9qL+mqx\niYRGhUeqnJYN6zBlfDpFJSWMzviCrXvyWbBuF098vJwR/VtzVtdmYUcUqdb05QWpkjo3T+G5MQMZ\n9eznjJs0i6Jip0m9RO6+8ODpAkUk3nTEI1XWgHaN+POo/ny1cQ+LN+3hd5f0okFdtdhEwqYjHqnS\nzu7anCevHsDa7fsY0r35kTcQkZhT4ZEq7xwVHJEKRa02ERGJKxUeERGJKxUeERGJKxUeERGJKxUe\nERGJKxUeERGJKxUeERGJKxUeERGJKzva69RXB2a2FVh9HC+RCuSUU5wwVZX9AO1LRVVV9qWq7Acc\n3760c/cjXuhKhScGzCzT3dPCznG8qsp+gPaloqoq+1JV9gPisy9qtYmISFyp8IiISFyp8MTG02EH\nKCdVZT9A+1JRVZV9qSr7AXHYF53jERGRuNIRj4iIxJUKzzEys6FmtsTMlpnZbWU8n2Rmfwue/8LM\n2sc/ZXSi2JexZrbVzOYFt+vCyHkkZpZhZlvMLPsQz5uZ/SnYzywz6x/vjNGKYl/ONLNdpT6Tu+Od\nMRpm1sbMPjSzr8xsoZn9tIx1KsXnEuW+VJbPpbaZzTKz+cG+3FvGOrH7Hebuuh3lDUgAlgMdgURg\nPtD9oHV+BDwZ3L8S+FvYuY9jX8YC/xd21ij25QygP5B9iOfPB94BDDgZ+CLszMexL2cCb4adM4r9\nOAHoH9xPAb4u499XpfhcotyXyvK5GJAc3K8FfAGcfNA6MfsdpiOeY5MOLHP3Fe5eALwEDD9oneHA\n5OD+VOC7ZmZxzBitaPalUnD3T4Dth1llODDFIz4HGprZCfFJd3Si2JdKwd03uvvc4P4e4Cug1UGr\nVYrPJcp9qRSC/9e5wcNawe3gE/4x+x2mwnNsWgFrSz1ex7f/Af5nHXcvAnYBTeKS7uhEsy8AI4I2\nyFQzaxOfaOUu2n2tLE4JWiXvmFmPsMMcSdCq6Ufkr+vSKt3ncph9gUryuZhZgpnNA7YA77r7IT+X\n8v4dpsJzbMqq+gf/tRDNOhVBNDn/AbR3997Ae/z3r6DKprJ8JtGYS2R6kj7AY8DrIec5LDNLBl4B\nfubuuw9+uoxNKuzncoR9qTSfi7sXu3tfoDWQbmY9D1olZp+LCs+xWQeU/qu/NbDhUOuYWU2gARWz\ndXLEfXH3be6eHzx8BhgQp2zlLZrPrVJw990HWiXu/jZQy8xSQ45VJjOrReQX9V/d/dUyVqk0n8uR\n9qUyfS4HuPtO4CNg6EFPxex3mArPsZkNdDazDmaWSOTE27SD1pkGjAnujwQ+8OAsXQVzxH05qN9+\nEZHedmU0DbgmGEV1MrDL3TeGHepYmFmLA/12M0sn8rO8LdxU3xZkfA74yt0fPcRqleJziWZfKtHn\n0tTMGgb36wBDgMUHrRaz32E1y+NFqht3LzKzG4HpREaFZbj7QjO7D8h092lE/oH+xcyWEfkr4crw\nEh9alPvyEzO7CCgisi9jQwt8GGb2IpFRRalmtg64h8hJU9z9SeBtIiOolgH7gHHhJD2yKPZlJPBD\nMysC9gNXVtA/bAYDo4EFwfkEgF8BbaHSfS7R7Etl+VxOACabWQKR4vh3d38zXr/DNHOBiIjElVpt\nIiISVyo8IiISVyo8IiISVyo8IiISVyo8IiISVyo8IiExs9wjr3XY7aeaWcfgfrKZPWVmy4PZhj8x\ns0Fmlhjc11cnpMJQ4RGphII5wBLcfUWw6Fki37Xo7O49iHzXKjWY+PV94IpQgoqUQYVHJGTBN/Yn\nmFm2mS0wsyuC5TXM7M/BEcybZva2mY0MNhsFvBGsdyIwCLjT3UsAgtnG3wrWfT1YX6RC0OG3SPgu\nBfoCfYBUYLaZfULkm/LtgV5AMyJTFWUE2wwGXgzu9wDmuXvxIV4/GxgYk+Qix0BHPCLhOw14MZgt\neDPwMZFCcRrwsruXuPsm4MNS25wAbI3mxYOCVGBmKeWcW+SYqPCIhO9QF9c63EW39gO1g/sLgT5m\ndrif5yQg7xiyiZQ7FR6R8H0CXBFcmKspkctezwI+JXIBvhpm1pzIpKEHfAV0AnD35UAmcG+pmZE7\nm9nw4H4TYKu7F8Zrh0QOR4VHJHyvAVnAfOAD4JagtfYKkWuiZANPEbna5a5gm7f4ZiG6DmgBLDOz\nBUSum3TgmjZnEZkBWqRC0OzUIhWYmSW7e25w1DILGOzum4JrqHwYPD7UoIIDr/EqcLu7L4lDZJEj\n0qg2kYrtzeCCXYnAb4IjIdx9v5ndA7QC1hxq4+Difq+r6EhFoiMeERGJK53jERGRuFLhERGRuFLh\nERGRuFLhERGRuFLhERGRuFLhERGRuPr/yeGaYr2blBoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a0db13a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 9\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lr_cv.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='lbfgs', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs\n",
    "# LogisticRegressionCV比GridSearchCV快\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.08350408, -0.08351795, -0.08351906, -0.08351913],\n",
       "        [-0.08855341, -0.08870812, -0.08872599, -0.08872602],\n",
       "        [-0.08314284, -0.08323216, -0.08324237, -0.0832424 ],\n",
       "        [-0.08221991, -0.08219267, -0.08219213, -0.08219205],\n",
       "        [-0.0807325 , -0.08076111, -0.08076424, -0.08076426]]),\n",
       " 1: array([[-0.31682273, -0.31682131, -0.31682132, -0.31682128],\n",
       "        [-0.32037305, -0.32040159, -0.32040171, -0.32040164],\n",
       "        [-0.3152482 , -0.31526742, -0.31526695, -0.31526741],\n",
       "        [-0.31668648, -0.316691  , -0.31669098, -0.31669127],\n",
       "        [-0.31926347, -0.31929601, -0.31929551, -0.31929597]]),\n",
       " 2: array([[-0.26141765, -0.26141826, -0.26141811, -0.26141815],\n",
       "        [-0.26187469, -0.2618833 , -0.26188354, -0.26188343],\n",
       "        [-0.26767706, -0.26772995, -0.26773643, -0.2677364 ],\n",
       "        [-0.26382824, -0.26383129, -0.26383095, -0.26383164],\n",
       "        [-0.26329748, -0.2632904 , -0.26329033, -0.26329027]]),\n",
       " 3: array([[-0.13075342, -0.13082462, -0.13083402, -0.13083356],\n",
       "        [-0.12745658, -0.12747923, -0.12748572, -0.12748583],\n",
       "        [-0.12532256, -0.12529508, -0.12529384, -0.12529401],\n",
       "        [-0.1272142 , -0.12721983, -0.12722219, -0.12722254],\n",
       "        [-0.13065085, -0.13075155, -0.13076971, -0.1307698 ]]),\n",
       " 4: array([[-0.01543393, -0.01565281, -0.01585401, -0.01589247],\n",
       "        [-0.01243413, -0.01218776, -0.01237381, -0.01241723],\n",
       "        [-0.01480769, -0.01461964, -0.01468087, -0.01468522],\n",
       "        [-0.01807401, -0.01877102, -0.01942143, -0.01959824],\n",
       "        [-0.01482531, -0.01401067, -0.01392566, -0.01392358]]),\n",
       " 5: array([[-0.11888338, -0.11892358, -0.11892683, -0.11892717],\n",
       "        [-0.11654856, -0.11662492, -0.11663241, -0.11663233],\n",
       "        [-0.10608199, -0.10607115, -0.10606985, -0.1060698 ],\n",
       "        [-0.1041375 , -0.10414867, -0.10415052, -0.10415056],\n",
       "        [-0.1074037 , -0.10739968, -0.1073989 , -0.10739905]]),\n",
       " 6: array([[-0.10058192, -0.10063913, -0.10064559, -0.10064553],\n",
       "        [-0.11310178, -0.11313876, -0.11314281, -0.11314315],\n",
       "        [-0.10451677, -0.10452842, -0.1045309 , -0.10453093],\n",
       "        [-0.10081854, -0.10082037, -0.10081977, -0.10081977],\n",
       "        [-0.10074561, -0.10075142, -0.10075228, -0.10075112]]),\n",
       " 7: array([[-0.10180119, -0.1017894 , -0.10178882, -0.1017887 ],\n",
       "        [-0.11021176, -0.11031779, -0.11032843, -0.1103284 ],\n",
       "        [-0.1000844 , -0.10008046, -0.1000799 , -0.10007998],\n",
       "        [-0.10343972, -0.10347751, -0.10348203, -0.103482  ],\n",
       "        [-0.10526902, -0.10531156, -0.1053159 , -0.10531572]]),\n",
       " 8: array([[-0.09444624, -0.09454003, -0.09455289, -0.09455286],\n",
       "        [-0.08329698, -0.08332694, -0.08333119, -0.08333101],\n",
       "        [-0.09286802, -0.09289625, -0.09289935, -0.09289932],\n",
       "        [-0.09001808, -0.09010389, -0.09011385, -0.09011408],\n",
       "        [-0.09094905, -0.09095012, -0.09095121, -0.09095111]])}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEKCAYAAAAiizNaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl4VOXZx/HvTVgFBJRFJGwKKiB7\nWAT3rdQKVosKgoIbra+ordW2drNa277udauKghYVUFEsrQt1t7ZFCCQQQHYVwg7KvoQk9/vHHHzH\nmMAJZObMJL/Pdc3FnHXuw0B+eZ5zznPM3REREUmWalEXICIiVYuCR0REkkrBIyIiSaXgERGRpFLw\niIhIUil4REQkqRQ8IiKSVAoeERFJKgWPiIgkVfWoC0hFjRs39jZt2kRdhohIWpk1a9ZGd29yoPUU\nPKVo06YN2dnZUZchIpJWzOyLMOupq01ERJJKwSMiIkml4BERkaRS8IiISFIpeEREJKkUPCIiklQK\nHhERSSrdxyMiUsUVFhWzaN02Zq/YTIYZl/VpldDPU/CIiFQxX+4oYPYXXzF7Rew1N38LOwuKAOjR\nqqGCR0REDl58ayYnCJvPN+0EoHo1o0Pzw7m4ZyY9WjeiR6tGZDaqk/CaFDwiIpVIfGsmZ8Vm5uRv\n/ro107heLXq0asiQ3q3o3rIhXTIbUqdmRtJrVPCIiKSp/bVmMqoZHUtpzZhZxFUreERE0saXOwrI\nCc7LzP6iZGumJt1bNeLSXq3o0Sq61kwYCh4RkRSUrq2ZMBQ8IiIpoLK0ZsJQ8IiIJFnJ1kzOys18\ntnEH8O3WTPeWjWh5RPq0ZsJQ8IiIJFiY1swlWS0rRWsmDAWPiEgFquqtmTAUPCIih0CtmfJLaPCY\n2QDgISADeNrd/7fE8lOBPwNdgCHuPjmY3xp4NdiuBvCIuz8RLPsAaA7sCnZzrruvN7MfAdcDRcB2\nYJS7LzCzI4HJQC/gWXcfncBDFpFKbF9rJmfF5q9v0CzZmhncM5MerWJXmlXF1kwYCQseM8sAHgPO\nAfKBmWY21d0XxK22AhgJ3FJi8zVAP3ffY2b1gHnBtquD5cPcPbvENhPiwmkQ8AAwANgN/AY4MXiJ\niISi1kxiJLLF0xtY6u7LAcxsEnAB8HXwuPvnwbLi+A3dvSBushYhHt/g7lvjJusCHszfAXxsZu0O\n6ihEpEooKnYWrd329cCZJVszHZrXV2umgiQyeFoAK+Om84E+YTc2s5bA60A74Na41g7AM2ZWBLwC\n3OXuHmxzPXAzUBM489DKF5HKrDytmc6ZDTispk6JV5RE/k2W9quAh93Y3VcCXczsaOA1M5vs7uuI\ndbOtMrP6xILncmB8sM1jwGNmdhnwa2BE6GLNRgGjAFq1SuyQ4CKSXGrNpJZEBk8+0DJuOhNYXca6\nZXL31WY2HzgFmOzuq4L528xsArEuvfElNpsEPF7OzxkDjAHIysoKHZAiknrCtGYuzooFTRe1ZpIu\nkX/bM4H2ZtYWWAUMAS4Ls6GZZQKb3H2XmTUC+gMPmFl1oKG7bzSzGsD5wDvBNu3dfUmwi+8BS0rb\nt4hULmrNpJ+EBY+7F5rZaGAascuix7n7fDO7E8h296lm1guYAjQCBprZHe7eCegA3G9mTqzL7j53\nzzOzusC0IHQyiIXOU8FHjjazs4G9wFfEdbOZ2efA4UBNM/s+sUuw46+uE5E0sa81s++S5jkrN7Mj\naM0cWVetmXRgwXl5iZOVleXZ2SWv1haRZAvTmtnXklFrJnpmNsvdsw60nn4VEJGU8dWOAnJWxs7L\nqDVTeelbE5FIhGnN/EDnZiolBY+IJN305Zu47vlZfLVzL6DWTFWjb1ZEkurfSzdy9V9nktnoMH47\nsCM9WjWi1RGHqTVThSh4RCRpPlq8gWvHZ9PmyLq8cG0fGterFXVJEoEDjoEmIlIR3l+0nmvGZ9O2\ncV0mKHSqNLV4RCTh3v10Hdc9P5v2zerx/NV9aFS3ZtQlSYQUPCKSUNPmr2X0hNl0aH44z13VhwaH\n1Yi6JImYutpEJGHezFvD9S/MptPRDXjuaoWOxKjFIyIJ8Y+5q7lpUi7dWjbk2St7Ub+2Qkdi1OIR\nkQr3t9xV3Dgxhx6tGvLXq3ordOQb1OIRkQr1yqx8bp08h95tj2DsiF7UraUfM/JN+hchIhXmpeyV\n/PyVufQ79kievqIXdWpmRF2SpCB1tYlIhZg4YwU/mzyXk9s1ZuwIhY6UTcEjIofsuelfcNureZx+\nfBOeuiKL2jUUOlI2dbWJyCF59t+f8bu/L+CsE5ryl+E9qFVdoSP7p+ARkYP29L+Wc9frn3Jux2Y8\nelkPalZXJ4ocmIJHRA7Kkx8u409vLuS7Jx7Fw0O7UyNDoSPhKHhEpNwee38p905bxPldmvPgpd0U\nOlIuCh4RKZeH3lnCg+8s5oJuR3P/xV2prtCRclLwiEgo7s6Dby/m4feWclGPFtw7uCsZ1fTwNik/\nBY+IHJC7c++0Rfzlg2VckpXJny7qotCRg6bgEZH9cnf+982FPPnRcob2bsUfvn8i1RQ6cggUPCJS\nJnfnrtc/ZezHn3F539bcMaiTQkcOmYJHRErl7tzx9wU8+5/PGdmvDbcP7IiZQkcOnYJHRL6luNj5\n7dR5PD99Bdec3JZffa+DQkcqjIJHRL6huNj51Wt5TJyxkh+edgy/GHCCQkcqlIJHRL5WVOz84pW5\nvDwrn9FntOOn5x6n0JEKp+ARESAWOre+PIdXc1Zx01nt+fHZ7RU6khAKHhGhsKiYm1+aw9Q5q7n5\nnOO48az2UZcklVhCx7owswFmtsjMlprZL0pZfqqZzTazQjMbHDe/tZnNMrNcM5tvZj+KW/ZBsM/c\n4NU0mP8jM8sL5n1sZh3jtrktqGGRmX0nkccskm72FhVz06Rcps5Zzc8GHK/QkYRLWIvHzDKAx4Bz\ngHxgpplNdfcFcautAEYCt5TYfA3Qz933mFk9YF6w7epg+TB3zy6xzQR3fyL47EHAA8CAIICGAJ2A\no4F3zOw4dy+qsIMVSVMFhcXcODGHt+av5ZfnncCoU4+NuiSpAhLZ4ukNLHX35e5eAEwCLohfwd0/\nd/e5QHGJ+QXuvieYrBWmTnffGjdZF/Dg/QXAJHff4+6fAUuD2kSqtD2FRVw/YTZvzV/Lb87vqNCR\npElk8LQAVsZN5wfzQjGzlmY2N9jH3XGtHYBngi6131jc2U8zu97MlgH3ADeWpw4zG2Vm2WaWvWHD\nhrBliqSl3XuLuO752by9YB13DOrE1Se3jbokqUISGTylXQ7jpcwrlbuvdPcuQDtghJk1CxYNc/fO\nwCnB6/K4bR5z92OBnwO/Lk8d7j7G3bPcPatJkyZhyxRJO7v3FvHD52bx3sL13PX9ExnRr03UJUkV\nk8jgyQdaxk1nAqvLWLdMQUtnPrGQwd1XBX9uAyZQerfZJOD7FVmHSGWwq6CIa8dn89GSDfzvRZ0Z\n3rd11CVJFZTI4JkJtDeztmZWk9gJ/qlhNjSzTDOrE7xvBPQHFplZdTNrHMyvAZwPzAum4y/F+R6w\nJHg/FRhiZrXMrC3QHphxyEcnkmZ2FhRy9V9n8vHSjdzzgy4M6d0q6pKkikrYVW3uXmhmo4FpQAYw\nzt3nm9mdQLa7TzWzXsAUoBEw0MzucPdOQAfgfjNzYl1l97l7npnVBaYFoZMBvAM8FXzkaDM7G9gL\nfAWMCOqYb2YvAQuAQuB6XdEmVc2OPYVc9exMZn7+JQ9c0pULu2dGXZJUYeYe+rRLlZGVleXZ2SWv\n1hZJT9v3FHLlMzOYvWIzD1zSlQu6hb7GR6RczGyWu2cdaD2NXCBSiW3dvZeR42YwJ38LDw/pzve6\nNI+6JBEFj0hltWXXXq4YN4P5q7bw2GXdGXCiQkdSg4JHpBLavLOAy8fOYOHarfxlWA/O7XRU1CWJ\nfE3BI1LJfLWjgGFPf8LS9dt58vKenHlCswNvJJJECh6RSmTT9j0Me/oTlm/cwZgrenL68U2jLknk\nWxQ8IpXEhm17GPb0dL7YtJOxI7I4pb1G4JDUpOARqQTWb93N0Kems3rzbp4Z2Yt+7RpHXZJImRQ8\nImlu7ZbdXPbUdNZu3c2zV/aizzFHRl2SyH4peETS2OrNuxj61HQ2btvD+Kt6k9XmiKhLEjkgBY9I\nmsr/aidDn5rO5h17GX91H3q2bhR1SSKhKHhE0tDKL3cyZMx0tu3ey3PX9KFby4ZRlyQSmoJHJM18\nsWkHQ8dMZ0dBES9c05fOmQ2iLkmkXBQ8Imnks42x0NlTWMSEa/vQ6WiFjqQfBY9Imli6fjuXPTWd\nwmJnwrV96dD88KhLEjkooR4EZ2b9g2fhYGbDzewBM9OjC0WSZMm6bQwZM51idyaNUuhIegv7BNLH\ngZ1m1hX4GfAFMD5hVYnI1xau3cqQMdMxg0mj+nJcs/pRlyRySMIGT6HHnhh3AfCQuz8E6F+/SIIt\nWL2VoWOmUz3DeHFUX9o11X87SX9hz/FsM7PbgOHAqWaWAdRIXFkiMm/VFoaP/YQ6NTKYeG1f2jSu\nG3VJIhUibIvnUmAPcLW7rwVaAPcmrCqRKm7Oys1c9tR06taszoujTlLoSKUSusVDrIutyMyOA04A\nJiauLJGqa/aKrxgxdgYN69ZgwjV9aXnEYVGXJFKhwrZ4PgJqmVkL4F3gSuDZRBUlUlXN+uJLrhg7\ngyPq1WTSqJMUOlIphQ0ec/edwEXAI+5+IdApcWWJVD0zPouFTpP6tXhx1Em0aFgn6pJEEiJ08JjZ\nScAw4PVgXkZiShKpev67bBMjxs3gqAa1eXFUX45qUDvqkkQSJmzw/Bi4DZji7vPN7Bjg/cSVJVJ1\n/HvpRq58dgaZjeowcVRfmh6u0JHKLdTFBe7+IfChmdU3s3ruvhy4MbGliVR+Hy7ewKjx2bRtXJfn\nr+lD43q1oi5JJOHCDpnT2cxygHnAAjObZWY6xyNyCN5fuJ5rx2dzTJN6TLi2r0JHqoywXW1PAje7\ne2t3bwX8FHgqcWWJVG7vLFjHD5+bxXHN6jHx2j4cUbdm1CWJJE3Y4Knr7l+f03H3DwDd0SZyEN6a\nt5brXphFh+b1eeHqvjQ8TKEjVUvYG0iXm9lvgOeC6eHAZ4kpSaTyeiNvDTdOzKFzZgP+elVvDq+t\nkaek6gnb4rkKaAK8CkwJ3l95oI3MbICZLTKzpWb2i1KWn2pms82s0MwGx81vHZxHyjWz+Wb2o7hl\nHwT7zA1eTYP5N5vZAjOba2bvxj+2wczuNrN5wevSkMcsUqH+Pmc1N0zMoVvLhoxX6EgVFvaqtq8o\n51VswUCijwHnAPnATDOb6u4L4lZbAYwEbimx+Rqgn7vvMbN6wLxg29XB8mHunl1imxwgy913mtl1\nwD3ApWb2PaAH0A2oRezqvDfdfWt5jkfkULyWs4qbX8olq80RPDOyF3Vr6RmMUnXt91+/mf0d8LKW\nu/ug/WzeG1gaXHqNmU0i9liFr4PH3T8PlhWX2G9B3GQtQrTM4s9BAdOJdQcCdAQ+dPdCoNDM5gAD\ngJcOtE+RijB5Vj63Tp5D37ZHMnZkFofVVOhI1Xag/wH3HcK+WwAr46bzgT5hNzazlsRGSWgH3BrX\n2gF4xsyKgFeAu4JnBcW7GngzeD8HuN3MHgAOA84gLvxEEumlmSv5+atz6X9sY566Ios6NTXgh8h+\ngye4cfRgWWm7DLuxu68EupjZ0cBrZjbZ3dcR62ZbZWb1iQXP5cQ9DdXMhgNZwGnBfv5pZr2A/wAb\ngP8Chd8q1mwUMAqgVatWYcsUKdOET1bwyyl5nHpcE8Zc3pPaNRQ6IhD+BtK84KR9/OtfZvagmR1Z\nxmb5QMu46UxgdRnrlilo6cwHTgmmVwV/bgMmEOvS21fn2cCvgEHuviduH39w927ufg6xQFxSyueM\ncfcsd89q0qRJecsU+Ybn/vs5v5ySxxnHK3RESgp7VdubxLq9hgWvvwP/AtZS9uMRZgLtzaytmdUE\nhgBTw3yYmWWaWZ3gfSOgP7DIzKqbWeNgfg3gfGKjKWBm3Ynd6DrI3dfH7StjXziaWRegC/DPkMct\nUm7P/PszfvO3+ZzdoRlPKHREviXsWc7+7t4/bjrPzP7t7v2Drq1vcfdCMxsNTCM2kvW4YIDRO4Fs\nd58adIFNARoBA83sDnfvBHQA7jczJ9ZCuc/d88ysLjAtCJ0M4B3+fwSFe4F6wMtmBrAiuPihBvCv\nYN5WYHhwoYFIhXv6X8u56/VP+U6nZjwytAc1q4f93U6k6ggbPPXMrI+7fwJgZr2J/ZCHUs6X7OPu\nbwBvlJj327j3M4l1wZXc7m1iLZOS83cAPcv4rLPLmL+b2JVtIgn1+AfLuPuthXyvc3P+PKQbNTIU\nOiKlCRs81wDjgntqjFjL4eqgBfKnRBUnki4efW8J9/1zMQO7Hs2Dl3SlukJHpExhbyCdCXQ2swbE\nnka6OW6x7oeRKu3P7yzmz+8s4cLuLbh3cBeFjsgBhAqeIHBuB04Npj8E7nT3LQmsTSSluTsPvL2Y\nR95byuCemdz9gy5kVCvtLgIRiRf2V7NxwDbgkuC1FXgmUUWJpDp3555pi3jkvaUM6dWSexQ6IqGF\nPcdzrLv/IG76DjPLTURBIqnO3fnTmwsZ89FyLuvTirsuOJFqCh2R0MK2eHaZ2cn7JsysP7ArMSWJ\npC53585/LGDMR8u54qTW/OH7Ch2R8grb4rkO+Ou+iwuAL4mNKi1SZbg7t0+dz/j/fsGV/dvw2/M7\nEtwfJiLlEPaqtlygq5kdHkzrkQJSpRQXO7/52zxe+GQFo049htu+e4JCR+QgHeixCDeXMR8Ad38g\nATWJpJTiYueXU/KYNHMl151+LD/7zvEKHZFDcKAWT/2kVCGSooqKnZ+/MpfJs/K54cx23HzOcQod\nkUN0oMci3JGsQkRSTWFRMbdOnsuUnFX85OzjuOns9lGXJFIplPsWazObnYhCRFJJYVExP3lpDlNy\nVnHLuQodkYp0MM/gVT+DVGp7i4r58aRcXs9bw88HnMB1px8bdUkilcrBBM/rFV6FSIooKCzmhomz\nmTZ/Hb/+XgeuOeWYqEsSqXTKHTzu/utEFCIStT2FRVz/wmze+XQ9tw/syJX920ZdkkilFHaQ0G2A\nl5i9BcgGfuruyyu6MJFk2r23iOuen8X7izZw5wWduOKkNlGXJFJphW3xPACsBiYQO8czBDgKWERs\nANHTE1GcSDLs3lvEqOdm8dHiDfzxws5c1qdV1CWJVGphr2ob4O5Puvs2d9/q7mOA89z9RWKPrRZJ\nS7sKirjmr9n8a8kG7vlBF4WOSBKEDZ5iM7vEzKoFr0vilpXsghNJCzsLCrnq2Zn8e9lG7h3clUt6\ntYy6JJEqIWzwDAMuB9YD64L3w82sDjA6QbWJJMz2PYWMfGYmn3y2iQcv6cbgnplRlyRSZYQdJHQ5\nMLCMxR9XXDkiibdt916ufGYmOSs389CQ7gzsenTUJYlUKaFaPGZ2nJm9a2bzgukuZqbLqiXtbN29\nlyvGzSB35WYeGarQEYlC2K62p4DbgL0A7j6X2JVtImljy869XP70J8xbtYXHhvXgvM7Noy5JpEoK\nezn1Ye4+o8SovIUJqEckITbvLGD42E9YvHY7jw/rydkdm0VdkkiVFTZ4NprZsQRXsJnZYGBNwqoS\nqUBf7ihg+NOfsHTDdp68vCdnnNA06pJEqrSwwXM9MAY4wcxWAZ8Ru9JNJKXlrtzM6Amz2bBtD09d\nkcVpxzWJuiSRKi9s8KwCngHeB44AtgIjgDsTVJfIIXF3xn78GXe/tZCm9WszcVRferTSvc4iqSBs\n8PwN2AzMJjZ0jkjK+mpHAbe8PId3F67n3I7NuHdwVxocViPqskQkEDZ4Mt19QEIrEakA2Z9/yQ0T\nc9i0vYDfDezIiH5t9KhqkRQTNnj+Y2ad3T0vodWIHKTiYufxD5fxwNuLyWxUh1eu60fnzAZRlyUi\npQh7H8/JwCwzW2Rmc80sz8zmHmgjMxsQbLPUzH5RyvJTzWy2mRUGV8rtm9/azGaZWa6ZzTezH8Ut\n+yDYZ27wahrMv9nMFgT1vWtmreO2uSfYz6dm9rDpV+BKZeP2PYx4Zgb3TlvEd088in/ccLJCRySF\nhW3xfLe8OzazDOAx4BwgH5hpZlPdfUHcaiuAkcAtJTZfA/Rz9z1mVg+YF2y77/zSMHfPLrFNDpDl\n7jvN7DrgHuBSM+sH9Ae6BOt9DJwGfFDeY5LU859lG7lpUi5bd+3ljxd2ZmjvlupaE0lxYcdq++Ig\n9t0bWLrvIXFmNgm4APg6eNz982BZcYnPK4ibrEWIlpm7vx83OR0Yvm8RUBuoSexZQjWIDXQqaayo\n2Hn43SU8/N4Sjmlcl/FX9aZD88OjLktEQij3o6/LoQWwMm46H+gTdmMzawm8DrQDbo1r7QA8Y2ZF\nwCvAXe5e8tEMVwNvArj7f83sfWKtKAMedfdPy3swkjrWbd3NTZNymL78Sy7q0YLfX3AidWsl8p+y\niFSkRP5vLa2/I/Sze9x9JdDFzI4GXjOzye6+jlg32yozq08seC4Hxn/9oWbDgSxi3WmYWTugA7Bv\n3Pu3zexUd//oG8WajQJGAbRqpYeBpaoPF2/g5hdz2VlQxH0Xd9XjDETSUNiLCw5GPhD/ZK1MDuIe\noKClMx84JZheFfy5jdijuHvvW9fMzgZ+BQxy9z3B7AuB6e6+3d23E2sJ9S3lc8a4e5a7ZzVporvb\nU83eomLufmshI8bNoHG9Wvz9hv4KHZE0lcjgmQm0N7O2ZlaT2GjWU8NsaGaZwUPmMLNGxC4OWGRm\n1c2scTC/BnA+sO9RDd2BJ4mFzvq43a0ATgu2rUGsJaSutjSyavMuhoyZzuMfLGNo71b8bXR/2jWt\nH3VZInKQEtbV5u6FZjYamAZkAOPcfb6Z3Qlku/tUM+sFTAEaAQPN7A5370Ssa+x+M3NiXXb3uXue\nmdUFpgUBkgG8Q+yRDQD3AvWAl4Ormla4+yBgMnAmkEesq+8td/97oo5bKtbbC9Zxy8tzYhcTDO3O\nID0/RyTt2bfPy0tWVpZnZ5e8WluSqaAw1rU29uPPOLHF4Tw6tAdtGteNuiwR2Q8zm+XuWQdaT5cC\nScpZsWknoyfOZm7+Fkb2a8Nt551AreoZUZclIhVEwSMp5Y28Nfx88lzM4InhPRlw4lFRlyQiFUzB\nIylh994i7np9Ac9PX0G3lg15ZGh3Wh5xWNRliUgCKHgkcss3bOf6CTl8umYrPzz1GG75zvHUyEjk\nBZciEiUFj0TqtZxV/HJKHrWqV2PcyCzOPKFZ1CWJSIIpeCQSOwsK+d3U+byUnU/vNkfw0NBuNG9Q\nJ+qyRCQJFDySdIvXbeP6F2azdMN2bjizHTed1Z7q6loTqTIUPJI07s7L2fn8duo86tWqwXNX9eHk\n9o2jLktEkkzBI0mxfU8hv5qSx99yV9O/3ZE8eGk3mtavHXVZIhIBBY8k3PzVWxg9IYcvNu3gp+cc\nx/+c0Y6ManpYm0hVpeCRhHF3np/+Bb9//VMaHVaDidf2pc8xR0ZdlohETMEjCbFl115ue3Uub+St\n5fTjm3D/xV05sl6tqMsSkRSg4JEKN2flZkZPnM2azbu57bsncO0px1BNXWsiElDwSIVxd8Z+/Bl3\nv7WQpvVr8+IPT6Jn60ZRlyUiKUbBIxXiqx0F3PLyHN5duJ5zOzbj3sFdaXBYjajLEpEUpOCRQ5b9\n+ZfcMDGHTdsL+N3Ajozo14bgYXwiIt+i4JGDVlzsPP7hMh54ezGZjerwynX96JzZIOqyRCTFKXjk\noGzcvoefvJjLv5Zs5PwuzfnTRZ2pX1tdayJyYAoeKbf/LNvITZNy2bprL3+8sDNDe7dU15qIhKbg\nkdCKip2H313Cw+8toW3juoy/qjcdmh8edVkikmYUPBLKuq27uWlSDtOXf8lFPVrw+wtOpG4t/fMR\nkfLTTw45oA8Xb+DmF3PZWVDEfRd3ZXDPzKhLEpE0puCRMu0tKuaBtxfz+AfLOL5ZfR4b1p12TetH\nXZaIpDkFj5Rq1eZd3Dgxh1lffMXQ3q24fWBHatfIiLosEakEFDzyLW8vWMctL8+hsKiYh4d2Z1DX\no6MuSUQqEQWPfK2gsJi731rI2I8/o9PRh/PoZT1o27hu1GWJSCWj4BEAVmzayeiJs5mbv4WR/dpw\n23knUKu6utZEpOIpeIQ38tbw88lzMYMnhvdkwIlHRV2SiFRiCp4qbPfeIu56fQHPT19Bt5YNeWRo\nd1oecVjUZYlIJafgqaKWb9jO9RNy+HTNVn546jHc8p3jqZFRLeqyRKQKSOhPGjMbYGaLzGypmf2i\nlOWnmtlsMys0s8Fx81ub2SwzyzWz+Wb2o7hlHwT7zA1eTYP5N5vZAjOba2bvmlnrYP4Zcevmmtlu\nM/t+Io871b2Ws4rzH/mYtVt2MW5kFred10GhIyJJk7AWj5llAI8B5wD5wEwzm+ruC+JWWwGMBG4p\nsfkaoJ+77zGzesC8YNvVwfJh7p5dYpscIMvdd5rZdcA9wKXu/j7QLajpCGAp8M8KO9A0squgiNun\nzuOl7Hx6tzmCh4Z2o3mDOlGXJSJVTCK72noDS919OYCZTQIuAL4OHnf/PFhWHL+huxfETdYiRMss\nCJh9pgPDS1ltMPCmu+8MdwiVx+J127j+hdks3bCdG85sx01ntae6WjkiEoFE/uRpAayMm84P5oVi\nZi3NbG6wj7vjWjsAzwTdZr+x0sfjvxp4s5T5Q4CJZXzeKDPLNrPsDRs2hC0z5bk7L81cyaBHP+ar\nnQWMv6o3Pz33eIWOiEQmkS2e0gLBw27s7iuBLmZ2NPCamU1293XEutlWmVl94BXgcmD81x9qNhzI\nAk77RjFmzYHOwLQyPm8MMAYgKysrdJ2pbPueQn49JY/XclfTv92RPHhpN5rWrx11WSJSxSXy1958\noGXcdCawuox1yxS0dOYDpwTTq4I/twETiHXpAWBmZwO/Aga5+54Su7oEmOLue8tbQzqav3oLAx/5\nmKlzVvPTc45j/FV9FDoikhISGTwzgfZm1tbMahLr5poaZkMzyzSzOsH7RkB/YJGZVTezxsH8GsD5\nwLxgujvwJLHQWV/KbodSRjcF2RdQAAAL2UlEQVRbZeLuPPffz7nwL/9hZ0EhE6/tyw1ntSejmp4Q\nKiKpIWFdbe5eaGajiXVtZQDj3H2+md0JZLv7VDPrBUwBGgEDzewOd+8EdADuNzMn1mV3n7vnmVld\nYFoQOhnAO8BTwUfeC9QDXg5O+6xw90EAZtaGWOvrw0QdbyrYsmsvt706lzfy1nL68U24/+KuHFmv\nVtRliYh8g7lXitMZFSorK8uzs0terZ3a5qzczOiJs1mzeTe3fud4rj3lGKqplSMiSWRms9w960Dr\naeSCNOfujP34M+5+ayFN69fmxR+eRM/WjaIuS0SkTAqeNPbVjgJunTyHdz5dz7kdm3Hv4K40OKxG\n1GWJiOyXgidNZX/+JTdOzGHj9gJ+N7AjI/q1ofRbmkREUouCJ80UFztPfLSM+/+5mBYN6/DKdf3o\nnNkg6rJEREJT8KSRjdv38JMXc/nXko18r0tz/nRRZw6vra41EUkvCp408Z9lG7lpUi5bd+3ljxd2\nZmjvlupaE5G0pOBJcUXFziPvLeHhd5fQpnFdxl/Vmw7ND4+6LBGRg6bgSWHrtu7mpkk5TF/+JRf1\naMHvLziRurX0lYlIetNPsRT14eIN3PxiLjsLirjv4q4M7pkZdUkiIhVCwZNi9hYV88Dbi3n8g2Uc\n36w+jw3rTrum9aMuS0Skwih4Usjqzbu4YWIOs774iqG9W3H7wI7UrpERdVkiIhVKwZMi3lmwjlsm\nz2FvYTEPD+3OoK5HR12SiEhCKHgiVlBYzN1vLWTsx5/R6ejDefSyHrRtXDfqskREEkbBE6EVm3Yy\neuJs5uZvYWS/Ntx23gnUqq6uNRGp3BQ8EXkjbw0/nzwXM3hieE8GnHhU1CWJiCSFgifJdu8t4q7X\nF/D89BV0a9mQR4Z2p+URh0VdlohI0ih4kmj5hu1cPyGHT9dsZdSpx3Drd46nRkYinz4uIpJ6FDxJ\n8lrOKn45JY9a1asxbmQWZ57QLOqSREQioeBJsF0FRdw+dR4vZefTu80RPDS0G80b1Im6LBGRyCh4\nEmjxum1c/8Jslm7Yzg1ntuOms9pTXV1rIlLFKXgSwN15OTuf306dR71a1Rl/VW9Oad8k6rJERFKC\ngqeCbd9TyK+n5PFa7mr6HXskfx7Sjab1a0ddlohIylDwVKCVX+5kxLgZfL5pBz895zj+54x2ZFTT\nw9pEROIpeCpQk/q1aNO4Ln+8qDN9jzky6nJERFKSgqcC1a6RwbiRvaIuQ0QkpekSKxERSSoFj4iI\nJJWCR0REkkrBIyIiSZXQ4DGzAWa2yMyWmtkvSll+qpnNNrNCMxscN7+1mc0ys1wzm29mP4pb9kGw\nz9zg1TSYf7OZLTCzuWb2rpm1jtumlZn908w+DdZpk8jjFhGRsiXsqjYzywAeA84B8oGZZjbV3RfE\nrbYCGAncUmLzNUA/d99jZvWAecG2q4Plw9w9u8Q2OUCWu+80s+uAe4BLg2XjgT+4+9vB/oor6DBF\nRKScEtni6Q0sdffl7l4ATAIuiF/B3T9397mUCAJ3L3D3PcFkrTB1uvv77r4zmJwOZAKYWUeguru/\nHay3PW49ERFJskQGTwtgZdx0fjAvFDNraWZzg33cHdfaAXgm6Gb7jZmVNjTA1cCbwfvjgM1m9qqZ\n5ZjZvUFrTEREIpDIG0hLCwQPu7G7rwS6mNnRwGtmNtnd1xHrZltlZvWBV4DLiXWlxT7UbDiQBZwW\nzKoOnAJ0J9a19yKx7r2x3yjWbBQwKpjcbmaLwtZaisbAxkPYPlVUluMAHUuqqizHUlmOAw7tWFof\neJXEBk8+0DJuOhNYXca6ZXL31WY2n1h4THb3VcH8bWY2gViX3ngAMzsb+BVwWlxXXT6Q4+7Lg3Ve\nA/pSInjcfQwwprz1lcbMst09qyL2FaXKchygY0lVleVYKstxQHKOJZFdbTOB9mbW1sxqAkOAqWE2\nNLNMM6sTvG8E9AcWmVl1M2sczK8BnA/MC6a7A08Cg9x9fYk6GpnZvucSnAnEX+AgIiJJlLDgcfdC\nYDQwDfgUeMnd55vZnWY2CMDMeplZPnAx8GTQsgHoAHxiZnOAD4H73D2P2IUG04JzP7nAKuCpYJt7\ngXrAy8H5n6lBHUXErpp718zyiHUB7ttGRESSLKGDhLr7G8AbJeb9Nu79TIKrz0qs8zbQpZT5O4Ce\nZXzW2fupo9T9JVCFdNmlgMpyHKBjSVWV5Vgqy3FAEo7F3EOf7xcRETlkGjJHRESSSsFzkEIMB1TL\nzF4Mln+SysP0hDiWkWa2IW6YomuiqPNAzGycma03s3llLDczezg4zrlm1iPZNYYV4lhON7Mtcd/J\nb0tbL2rB/XjvB8NVzTezm0pZJy2+l5DHki7fS20zm2Fmc4JjuaOUdRL3M8zd9SrnC8gAlgHHADWB\nOUDHEuv8D/BE8H4I8GLUdR/CsYwEHo261hDHcirQA5hXxvLziN1YbMQuqf8k6poP4VhOB/4RdZ0h\njqM50CN4Xx9YXMq/r7T4XkIeS7p8LwbUC97XAD4B+pZYJ2E/w9TiOTgHHA4omP5r8H4ycFYZoyxE\nLcyxpAV3/wj4cj+rXACM95jpQEMza56c6sonxLGkBXdf4+6zg/fbiF3hWnIEk7T4XkIeS1oI/q63\nB5M1glfJE/4J+xmm4Dk4YYYD+nodj11avgU4MinVlU/YoY1+EHSDTDazlqUsTweHNIxTCjop6Cp5\n08w6RV3MgQRdNd2J/XYdL+2+l/0cC6TJ92JmGWaWC6wH3nb3Mr+Xiv4ZpuA5OGGGAzqkIYOSKEyd\nfwfauHsX4B3+/7egdJMu30kYs4HW7t4VeAR4LeJ69stio8K/AvzY3beWXFzKJin7vRzgWNLme3H3\nInfvRuyWlt5mdmKJVRL2vSh4Dk6Y4YC+XsfMqgMNSM2ukwMei7tv8v8fgugpyriXKg1UyDBOqcDd\nt+7rKvHY/XI19o3qkWqCUUZeAV5w91dLWSVtvpcDHUs6fS/7uPtm4ANgQIlFCfsZpuA5OGGGA5oK\njAjeDwbe8+AsXYo54LGU6G8fRKxvOx1NBa4IrqLqC2xx9zVRF3UwzOyoff3tZtab2P/lTdFW9W1B\njWOBT939gTJWS4vvJcyxpNH30sTMGgbv6wBnAwtLrJawn2EJHbmgsnL3QjPbNxxQBjDOg+GAgGx3\nn0rsH+hzZraU2G8JQ6KruGwhj+VGiw1zVEjsWEZGVvB+mNlEYlcVNbbYUEy3Eztpirs/QWwUjfOA\npcBO4MpoKj2wEMcyGLjOzAqBXcCQFP3Fpj+xEeTzgvMJAL8EWkHafS9hjiVdvpfmwF8t9oiYasSG\nNPtHsn6GaeQCERFJKnW1iYhIUil4REQkqRQ8IiKSVAoeERFJKgWPiIgklYJHJCJmtv3Aa+13+8lm\ndkzwvp6ZPWlmy4LRhj8ysz5mVjN4r1snJGUoeETSUDAGWIa7Lw9mPU3sXov27t6J2L1WjYOBX98F\nLo2kUJFSKHhEIhbcsX+vmc0zszwzuzSYX83M/hK0YP5hZm+Y2eBgs2HA34L1jgX6AL9292KAYLTx\n14N1XwvWF0kJan6LRO8ioBvQFWgMzDSzj4jdKd8G6Aw0JTZU0bhgm/7AxOB9JyDX3YvK2P88oFdC\nKhc5CGrxiETvZGBiMFrwOuBDYkFxMvCyuxe7+1rg/bhtmgMbwuw8CKQCM6tfwXWLHBQFj0j0ynq4\n1v4eurULqB28nw90NbP9/X+uBew+iNpEKpyCRyR6HwGXBg/makLssdczgI+JPYCvmpk1IzZo6D6f\nAu0A3H0ZkA3cETcycnszuyB4fySwwd33JuuARPZHwSMSvSnAXGAO8B7ws6Br7RViz0SZBzxJ7GmX\nW4JtXuebQXQNcBSw1MzyiD03ad8zbc4gNgK0SErQ6NQiKczM6rn79qDVMgPo7+5rg2eovB9Ml3VR\nwb59vArc5u6LklCyyAHpqjaR1PaP4IFdNYHfBy0h3H2Xmd0OtABWlLVx8HC/1xQ6kkrU4hERkaTS\nOR4REUkqBY+IiCSVgkdERJJKwSMiIkml4BERkaRS8IiISFL9H/91PHCwHCYNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a0f158950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dict with classes as the keys, and the values as the grid of scores obtained during cross-validating each fold,\n",
    "# Each dict value has shape (n_folds, len(Cs))\n",
    "n_Cs = len(Cs)\n",
    "n_classes = 9\n",
    "scores =  np.zeros((n_classes,n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "        scores[j][:] = np.mean(lrcv_L2.scores_[j],axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs,1)) \n",
    "#plt.plot(np.log10(reg.Cs)*np.ones(3), [0.28, 0.29, 0.30])\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()\n",
    "\n",
    "#print ('C is:',lr_cv.C_)  #对多类分类问题，每个类别的分类器有一个C\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
